From cc6539f4b9019f2bb025d5828c741c9c4993e57b Mon Sep 17 00:00:00 2001 From: George Lu Date: Mon, 11 Jun 2018 10:59:05 -0400 Subject: [PATCH 001/372] Requested changes Remove g_displaylevel/setNotificationLevel function Add extern "C" Remove averaging Reorder arguments --- tests/paramgrill.c | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/paramgrill.c b/tests/paramgrill.c index db45220c3..2d7e52a43 100644 --- a/tests/paramgrill.c +++ b/tests/paramgrill.c @@ -170,7 +170,6 @@ BMK_benchParam(BMK_result_t* resultPtr, ZSTD_CCtx* ctx, ZSTD_DCtx* dctx, const ZSTD_compressionParameters cParams) { - BMK_return_t res = BMK_benchMem(srcBuffer,srcSize, &srcSize, 1, 0, &cParams, NULL, 0, ctx, dctx, 0, "File"); *resultPtr = res.result; return res.errorCode; From 20f4f3237911288d7e1af7e61dc7d8a5ab008cda Mon Sep 17 00:00:00 2001 From: George Lu Date: Tue, 12 Jun 2018 15:54:43 -0400 Subject: [PATCH 002/372] Add to bench -Remove global variables -Remove gv setting functions -Add advancedParams struct -Add defaultAdvancedParams(); -Change return type of bench Files -Change cli to use new interface -Changed error returns to own struct value -Change default compression benchmark to use decompress_generic -Add CustomBench function -Add Documentation for new functions --- programs/bench.c | 916 ++++++++++++++++++++++++++++----------------- programs/bench.h | 137 +++++-- programs/zstdcli.c | 27 +- tests/paramgrill.c | 4 +- 4 files changed, 692 insertions(+), 392 deletions(-) diff --git a/programs/bench.c b/programs/bench.c index 09697d1fe..7b9ea8218 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -64,9 +64,10 @@ static const size_t maxMemory = (sizeof(size_t)==4) ? (2 GB - 64 MB) : (size_t)(1ULL << ((sizeof(size_t)*8)-31)); +//TODO: remove this gv as well +//Only used in Synthetic test. Separate? static U32 g_compressibilityDefault = 50; - /* ************************************* * console display ***************************************/ @@ -90,88 +91,51 @@ static UTIL_time_t g_displayClock = UTIL_TIME_INITIALIZER; # define DEBUG 0 #endif #define DEBUGOUTPUT(...) { if (DEBUG) DISPLAY(__VA_ARGS__); } -#define EXM_THROW(error, ...) { \ + +#define EXM_THROW_INT(errorNum, ...) { \ DEBUGOUTPUT("%s: %i: \n", __FILE__, __LINE__); \ - DISPLAYLEVEL(1, "Error %i : ", error); \ + DISPLAYLEVEL(1, "Error %i : ", errorNum); \ DISPLAYLEVEL(1, __VA_ARGS__); \ DISPLAYLEVEL(1, " \n"); \ - exit(error); \ + return errorNum; \ } +#define EXM_THROW(errorNum, retType, ...) { \ + retType r; \ + memset(&r, 0, sizeof(retType)); \ + DEBUGOUTPUT("%s: %i: \n", __FILE__, __LINE__); \ + DISPLAYLEVEL(1, "Error %i : ", errorNum); \ + DISPLAYLEVEL(1, __VA_ARGS__); \ + DISPLAYLEVEL(1, " \n"); \ + r.error = errorNum; \ + return r; \ +} /* ************************************* * Benchmark Parameters ***************************************/ -static int g_additionalParam = 0; -static U32 g_decodeOnly = 0; - -void BMK_setAdditionalParam(int additionalParam) { g_additionalParam=additionalParam; } - - -//TODO : Deal with DISPLAYLEVEL for all these set functions - -static U32 g_nbSeconds = BMK_TIMETEST_DEFAULT_S; - -void BMK_setNbSeconds(unsigned nbSeconds) -{ - g_nbSeconds = nbSeconds; - DISPLAY("- test >= %u seconds per compression / decompression - \n", g_nbSeconds); -} - -static size_t g_blockSize = 0; - -void BMK_setBlockSize(size_t blockSize) -{ - g_blockSize = blockSize; - if (g_blockSize) DISPLAY("using blocks of size %u KB \n", (U32)(blockSize>>10)); -} - -void BMK_setDecodeOnlyMode(unsigned decodeFlag) { g_decodeOnly = (decodeFlag>0); } - -static U32 g_nbWorkers = 0; - -void BMK_setNbWorkers(unsigned nbWorkers) { -#ifndef ZSTD_MULTITHREAD - if (nbWorkers > 0) DISPLAY("Note : multi-threading is disabled \n"); -#endif - g_nbWorkers = nbWorkers; -} - -static U32 g_realTime = 0; -void BMK_setRealTime(unsigned priority) { - g_realTime = (priority>0); -} - -static U32 g_separateFiles = 0; -void BMK_setSeparateFiles(unsigned separate) { - g_separateFiles = (separate>0); -} - -static U32 g_ldmFlag = 0; -void BMK_setLdmFlag(unsigned ldmFlag) { - g_ldmFlag = ldmFlag; -} - -static U32 g_ldmMinMatch = 0; -void BMK_setLdmMinMatch(unsigned ldmMinMatch) { - g_ldmMinMatch = ldmMinMatch; -} - -static U32 g_ldmHashLog = 0; -void BMK_setLdmHashLog(unsigned ldmHashLog) { - g_ldmHashLog = ldmHashLog; -} #define BMK_LDM_PARAM_NOTSET 9999 -static U32 g_ldmBucketSizeLog = BMK_LDM_PARAM_NOTSET; -void BMK_setLdmBucketSizeLog(unsigned ldmBucketSizeLog) { - g_ldmBucketSizeLog = ldmBucketSizeLog; + +BMK_advancedParams_t BMK_defaultAdvancedParams(void) { + BMK_advancedParams_t res = { + 0, /* mode */ + 0, /* nbCycles */ + BMK_TIMETEST_DEFAULT_S, /* nbSeconds */ + 0, /* blockSize */ + 0, /* nbWorkers */ + 0, /* realTime */ + 1, /* separateFiles */ + 0, /* additionalParam */ + 0, /* ldmFlag */ + 0, /* ldmMinMatch */ + 0, /* ldmHashLog */ + BMK_LDM_PARAM_NOTSET, /* ldmBuckSizeLog */ + BMK_LDM_PARAM_NOTSET /* ldmHashEveryLog */ + }; + return res; } -static U32 g_ldmHashEveryLog = BMK_LDM_PARAM_NOTSET; -void BMK_setLdmHashEveryLog(unsigned ldmHashEveryLog) { - g_ldmHashEveryLog = ldmHashEveryLog; -} /* ******************************************************** * Bench functions @@ -191,20 +155,264 @@ typedef struct { #define MIN(a,b) ((a) < (b) ? (a) : (b)) #define MAX(a,b) ((a) > (b) ? (a) : (b)) -BMK_return_t BMK_benchMem(const void* srcBuffer, size_t srcSize, +static void BMK_initCCtx(ZSTD_CCtx* ctx, + const void* dictBuffer, size_t dictBufferSize, int cLevel, + const ZSTD_compressionParameters* comprParams, const BMK_advancedParams_t* adv) { + if (adv->nbWorkers==1) { + ZSTD_CCtx_setParameter(ctx, ZSTD_p_nbWorkers, 0); + } else { + ZSTD_CCtx_setParameter(ctx, ZSTD_p_nbWorkers, adv->nbWorkers); + } + ZSTD_CCtx_setParameter(ctx, ZSTD_p_compressionLevel, cLevel); + ZSTD_CCtx_setParameter(ctx, ZSTD_p_enableLongDistanceMatching, adv->ldmFlag); + ZSTD_CCtx_setParameter(ctx, ZSTD_p_ldmMinMatch, adv->ldmMinMatch); + ZSTD_CCtx_setParameter(ctx, ZSTD_p_ldmHashLog, adv->ldmHashLog); + if (adv->ldmBucketSizeLog != BMK_LDM_PARAM_NOTSET) { + ZSTD_CCtx_setParameter(ctx, ZSTD_p_ldmBucketSizeLog, adv->ldmBucketSizeLog); + } + if (adv->ldmHashEveryLog != BMK_LDM_PARAM_NOTSET) { + ZSTD_CCtx_setParameter(ctx, ZSTD_p_ldmHashEveryLog, adv->ldmHashEveryLog); + } + ZSTD_CCtx_setParameter(ctx, ZSTD_p_windowLog, comprParams->windowLog); + ZSTD_CCtx_setParameter(ctx, ZSTD_p_hashLog, comprParams->hashLog); + ZSTD_CCtx_setParameter(ctx, ZSTD_p_chainLog, comprParams->chainLog); + ZSTD_CCtx_setParameter(ctx, ZSTD_p_searchLog, comprParams->searchLog); + ZSTD_CCtx_setParameter(ctx, ZSTD_p_minMatch, comprParams->searchLength); + ZSTD_CCtx_setParameter(ctx, ZSTD_p_targetLength, comprParams->targetLength); + ZSTD_CCtx_setParameter(ctx, ZSTD_p_compressionStrategy, comprParams->strategy); + ZSTD_CCtx_loadDictionary(ctx, dictBuffer, dictBufferSize); +} + + +static void BMK_initDCtx(ZSTD_DCtx* dctx, + const void* dictBuffer, size_t dictBufferSize) { + ZSTD_DCtx_loadDictionary(dctx, dictBuffer, dictBufferSize); +} + +typedef struct { + ZSTD_CCtx* ctx; + const void* dictBuffer; + size_t dictBufferSize; + int cLevel; + const ZSTD_compressionParameters* comprParams; + const BMK_advancedParams_t* adv; +} BMK_initCCtxArgs; + +static size_t local_initCCtx(void* payload) { + BMK_initCCtxArgs* ag = (BMK_initCCtxArgs*)payload; + BMK_initCCtx(ag->ctx, ag->dictBuffer, ag->dictBufferSize, ag->cLevel, ag->comprParams, ag->adv); + return 0; +} + +typedef struct { + ZSTD_DCtx* dctx; + const void* dictBuffer; + size_t dictBufferSize; +} BMK_initDCtxArgs; + +static size_t local_initDCtx(void* payload) { + BMK_initDCtxArgs* ag = (BMK_initDCtxArgs*)payload; + BMK_initDCtx(ag->dctx, ag->dictBuffer, ag->dictBufferSize); + return 0; +} + +/* additional argument is just the context */ +static size_t local_defaultCompress( + const void* srcBuffer, size_t srcSize, + void* dstBuffer, size_t dstSize, + void* addArgs) { + size_t moreToFlush = 1; + ZSTD_CCtx* ctx = (ZSTD_CCtx*)addArgs; + ZSTD_inBuffer in; + ZSTD_outBuffer out; + in.src = srcBuffer; + in.size = srcSize; + in.pos = 0; + out.dst = dstBuffer; + out.size = dstSize; + out.pos = 0; + while (moreToFlush) { + moreToFlush = ZSTD_compress_generic(ctx, &out, &in, ZSTD_e_end); + if (ZSTD_isError(moreToFlush)) { + return moreToFlush; + } + } + return out.pos; +} + +/* addiional argument is just the context */ +static size_t local_defaultDecompress( + const void* srcBuffer, size_t srcSize, + void* dstBuffer, size_t dstSize, + void* addArgs) { + size_t moreToFlush = 1; + ZSTD_DCtx* dctx = (ZSTD_DCtx*)addArgs; + ZSTD_inBuffer in; + ZSTD_outBuffer out; + in.src = srcBuffer; + in.size = srcSize; + in.pos = 0; + out.dst = dstBuffer; + out.size = dstSize; + out.pos = 0; + while (moreToFlush) { + moreToFlush = ZSTD_decompress_generic(dctx, + &out, &in); + if (ZSTD_isError(moreToFlush)) { + return moreToFlush; + } + } + return out.pos; + +} + +//ignore above for error stuff, return type still undecided + +/* mode 0 : iter = # seconds, else iter = # cycles */ +/* initFn will be measured once, bench fn will be measured x times */ +/* benchFn should return error value or out Size */ +//problem : how to get cSize this way for ratio? +//also possible fastest rounds down to 0 if 0 < loopDuration < nbLoops (that would mean <1ns / op though) +/* takes # of blocks and list of size & stuff for each. */ +BMK_customReturn_t BMK_benchCustom( + const char* functionName, size_t blockCount, + const void* const * const srcBuffers, size_t* srcSizes, + void* const * const dstBuffers, size_t* dstSizes, + size_t (*initFn)(void*), size_t (*benchFn)(const void*, size_t, void*, size_t, void*), + void* initPayload, void* benchPayload, + unsigned mode, unsigned iter, + int displayLevel) { + size_t srcSize = 0, dstSize = 0, ind = 0; + unsigned toAdd = 1; + + BMK_customReturn_t retval; + U64 totalTime = 0, fastest = (U64)(-1LL); + UTIL_time_t clockStart; + + { + unsigned i; + for(i = 0; i < blockCount; i++) { + memset(dstBuffers[i], 0xE5, dstSizes[i]); /* warm up and erase result buffer */ + } + + UTIL_sleepMilli(5); /* give processor time to other processes */ + UTIL_waitForNextTick(); + } + + /* display last 17 char's of functionName*/ + if (strlen(functionName)>17) functionName += strlen(functionName)-17; + if(!iter) { + if(mode) { + EXM_THROW(1, BMK_customReturn_t, "nbSeconds must be nonzero \n"); + } else { + EXM_THROW(1, BMK_customReturn_t, "nbLoops must be nonzero \n"); + } + + } + + for(ind = 0; ind < blockCount; ind++) { + srcSize += srcSizes[ind]; + } + + //change to switch if more modes? + if(!mode) { + int completed = 0; + U64 const maxTime = (iter * TIMELOOP_NANOSEC) + 1; + unsigned nbLoops = 1; + UTIL_time_t coolTime = UTIL_getTime(); + while(!completed) { + unsigned i, j; + /* Overheat protection */ + if (UTIL_clockSpanMicro(coolTime) > ACTIVEPERIOD_MICROSEC) { + DISPLAYLEVEL(2, "\rcooling down ... \r"); + UTIL_sleep(COOLPERIOD_SEC); + coolTime = UTIL_getTime(); + } + + for(i = 0; i < blockCount; i++) { + memset(dstBuffers[i], 0xD6, dstSizes[i]); /* warm up and erase result buffer */ + } + + clockStart = UTIL_getTime(); + (*initFn)(initPayload); + + for(i = 0; i < nbLoops; i++) { + for(j = 0; j < blockCount; j++) { + size_t res = (*benchFn)(srcBuffers[j], srcSizes[j], dstBuffers[j], dstSizes[j], benchPayload); + if(ZSTD_isError(res)) { + EXM_THROW(2, BMK_customReturn_t, "%s() failed on block %u of size %u : %s \n", + functionName, j, (U32)dstSizes[j], ZSTD_getErrorName(res)); + } else if (toAdd) { + dstSize += res; + } + } + toAdd = 0; + } + { U64 const loopDuration = UTIL_clockSpanNano(clockStart); + if (loopDuration > 0) { + fastest = MIN(fastest, loopDuration / nbLoops); + nbLoops = (U32)(TIMELOOP_NANOSEC / fastest) + 1; + } else { + assert(nbLoops < 40000000); /* avoid overflow */ + nbLoops *= 100; + } + totalTime += loopDuration; + completed = (totalTime >= maxTime); + } + } + } else { + unsigned i, j; + clockStart = UTIL_getTime(); + for(i = 0; i < iter; i++) { + for(j = 0; j < blockCount; j++) { + size_t res = (*benchFn)(srcBuffers[j], srcSizes[j], dstBuffers[j], dstSizes[j], benchPayload); + if(ZSTD_isError(res)) { + EXM_THROW(2, BMK_customReturn_t, "%s() failed on block %u of size %u : %s \n", + functionName, j, (U32)dstSizes[j], ZSTD_getErrorName(res)); + } else if(toAdd) { + dstSize += res; + } + } + toAdd = 0; + } + totalTime = UTIL_clockSpanNano(clockStart); + if(!totalTime) { + EXM_THROW(3, BMK_customReturn_t, "Cycle count (%u) too short to measure \n", iter); + } else { + fastest = totalTime / iter; + } + } + retval.error = 0; + retval.result.time = fastest; + retval.result.size = dstSize; + return retval; +} + +BMK_return_t BMK_benchMemAdvanced(const void* srcBuffer, size_t srcSize, const size_t* fileSizes, unsigned nbFiles, const int cLevel, const ZSTD_compressionParameters* comprParams, const void* dictBuffer, size_t dictBufferSize, ZSTD_CCtx* ctx, ZSTD_DCtx* dctx, - int displayLevel, const char* displayName) + int displayLevel, const char* displayName, const BMK_advancedParams_t* adv) { - size_t const blockSize = ((g_blockSize>=32 && !g_decodeOnly) ? g_blockSize : srcSize) + (!srcSize) /* avoid div by 0 */ ; + size_t const blockSize = ((adv->blockSize>=32 && (adv->mode != BMK_DECODE_ONLY)) ? adv->blockSize : srcSize) + (!srcSize) /* avoid div by 0 */ ; U32 const maxNbBlocks = (U32) ((srcSize + (blockSize-1)) / blockSize) + nbFiles; - blockParam_t* const blockTable = (blockParam_t*) malloc(maxNbBlocks * sizeof(blockParam_t)); - size_t const maxCompressedSize = ZSTD_compressBound(srcSize) + (maxNbBlocks * 1024); /* add some room for safety */ - void* const compressedBuffer = malloc(maxCompressedSize); + + /* these are the blockTable parameters, just split up */ + const void ** const srcPtrs = malloc(maxNbBlocks * sizeof(void*)); + size_t* const srcSizes = malloc(maxNbBlocks * sizeof(size_t)); + + void ** const cPtrs = malloc(maxNbBlocks * sizeof(void*)); + size_t* const cSizes = malloc(maxNbBlocks * sizeof(size_t)); + + void ** const resPtrs = malloc(maxNbBlocks * sizeof(void*)); + size_t* const resSizes = malloc(maxNbBlocks * sizeof(size_t)); + + const size_t maxCompressedSize = ZSTD_compressBound(srcSize) + (maxNbBlocks * 1024); /* add some room for safety */ + void* compressedBuffer = malloc(maxCompressedSize); void* resultBuffer = malloc(srcSize); + BMK_return_t results; size_t const loadedCompressedSize = srcSize; @@ -213,317 +421,242 @@ BMK_return_t BMK_benchMem(const void* srcBuffer, size_t srcSize, U32 nbBlocks; /* checks */ - if (!compressedBuffer || !resultBuffer || !blockTable) - EXM_THROW(31, "allocation error : not enough memory"); + if (!compressedBuffer || !resultBuffer || + !srcPtrs || !srcSizes || !cPtrs || !cSizes || !resPtrs || !resSizes) + EXM_THROW(31, BMK_return_t, "allocation error : not enough memory"); if(!ctx || !dctx) - EXM_THROW(31, "error: passed in null context"); + EXM_THROW(31, BMK_return_t, "error: passed in null context"); /* init */ if (strlen(displayName)>17) displayName += strlen(displayName)-17; /* display last 17 characters */ - if (g_nbWorkers==1) g_nbWorkers=0; /* prefer synchronous mode */ - - if (g_decodeOnly) { /* benchmark only decompression : source must be already compressed */ + if (adv->mode == BMK_DECODE_ONLY) { /* benchmark only decompression : source must be already compressed */ const char* srcPtr = (const char*)srcBuffer; U64 totalDSize64 = 0; U32 fileNb; for (fileNb=0; fileNb decodedSize) EXM_THROW(32, "original size is too large"); /* size_t overflow */ + if (totalDSize64 > decodedSize) EXM_THROW(32, BMK_return_t, "original size is too large"); /* size_t overflow */ free(resultBuffer); resultBuffer = malloc(decodedSize); - if (!resultBuffer) EXM_THROW(33, "not enough memory"); + if (!resultBuffer) EXM_THROW(33, BMK_return_t, "not enough memory"); cSize = srcSize; srcSize = decodedSize; ratio = (double)srcSize / (double)cSize; - } } + } + } - /* Init blockTable data */ + /* Init data blocks */ { const char* srcPtr = (const char*)srcBuffer; char* cPtr = (char*)compressedBuffer; char* resPtr = (char*)resultBuffer; U32 fileNb; for (nbBlocks=0, fileNb=0; fileNbmode == BMK_DECODE_ONLY) ? 1 : (U32)((remaining + (blockSize-1)) / blockSize); U32 const blockEnd = nbBlocks + nbBlocksforThisFile; for ( ; nbBlocksmode == BMK_DECODE_ONLY) ? thisBlockSize : ZSTD_compressBound(thisBlockSize); + //blockTable[nbBlocks].cSize = blockTable[nbBlocks].cRoom; + resPtrs[nbBlocks] = (void*)resPtr; + resSizes[nbBlocks] = (adv->mode == BMK_DECODE_ONLY) ? (size_t) ZSTD_findDecompressedSize(srcPtr, thisBlockSize) : thisBlockSize; srcPtr += thisBlockSize; - cPtr += blockTable[nbBlocks].cRoom; + cPtr += cSizes[nbBlocks]; //blockTable[nbBlocks].cRoom; resPtr += thisBlockSize; remaining -= thisBlockSize; - } } } + } + } + } /* warmimg up memory */ - if (g_decodeOnly) { + if (adv->mode == BMK_DECODE_ONLY) { memcpy(compressedBuffer, srcBuffer, loadedCompressedSize); } else { RDG_genBuffer(compressedBuffer, maxCompressedSize, 0.10, 0.50, 1); } /* Bench */ - { U64 fastestC = (U64)(-1LL), fastestD = (U64)(-1LL); - U64 const crcOrig = g_decodeOnly ? 0 : XXH64(srcBuffer, srcSize, 0); - UTIL_time_t coolTime; - U64 const maxTime = (g_nbSeconds * TIMELOOP_NANOSEC) + 1; - U32 nbDecodeLoops = (U32)((100 MB) / (srcSize+1)) + 1; /* initial conservative speed estimate */ - U32 nbCompressionLoops = (U32)((2 MB) / (srcSize+1)) + 1; /* initial conservative speed estimate */ - U64 totalCTime=0, totalDTime=0; - U32 cCompleted=g_decodeOnly, dCompleted=0; + + //TODO: Make sure w/o new loop decode_only code isn't run + //TODO: Support nbLoops and nbSeconds + { + U64 const crcOrig = (adv->mode == BMK_DECODE_ONLY) ? 0 : XXH64(srcBuffer, srcSize, 0); # define NB_MARKS 4 const char* const marks[NB_MARKS] = { " |", " /", " =", "\\" }; U32 markNb = 0; - - coolTime = UTIL_getTime(); DISPLAYLEVEL(2, "\r%79s\r", ""); - while (!cCompleted || !dCompleted) { - /* overheat protection */ - if (UTIL_clockSpanMicro(coolTime) > ACTIVEPERIOD_MICROSEC) { - DISPLAYLEVEL(2, "\rcooling down ... \r"); - UTIL_sleep(COOLPERIOD_SEC); - coolTime = UTIL_getTime(); + if (adv->mode != BMK_DECODE_ONLY) { + BMK_initCCtxArgs cctxprep = { ctx, dictBuffer, dictBufferSize, cLevel, comprParams, adv }; + BMK_customReturn_t compressionResults; + /* Compression */ + DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->\r", marks[markNb], displayName, (U32)srcSize); + compressionResults = BMK_benchCustom("ZSTD_compress_generic", nbBlocks, + srcPtrs, srcSizes, cPtrs, cSizes, + &local_initCCtx, &local_defaultCompress, + (void*)&cctxprep, (void*)(ctx), + adv->loopMode, adv->nbSeconds, displayLevel); + + if(compressionResults.error) { + results.error = compressionResults.error; + return results; } - if (!g_decodeOnly) { - /* Compression */ - DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->\r", marks[markNb], displayName, (U32)srcSize); - if (!cCompleted) memset(compressedBuffer, 0xE5, maxCompressedSize); /* warm up and erase result buffer */ + results.result.cSize = compressionResults.result.size; + ratio = (double)srcSize / (double)results.result.cSize; + markNb = (markNb+1) % NB_MARKS; + { + int const ratioAccuracy = (ratio < 10.) ? 3 : 2; + double const compressionSpeed = ((double)srcSize / compressionResults.result.time) * 1000; + int const cSpeedAccuracy = (compressionSpeed < 10.) ? 2 : 1; + results.result.cSpeed = compressionSpeed * 1000000; + DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.*f),%6.*f MB/s\r", + marks[markNb], displayName, (U32)srcSize, (U32)results.result.cSize, + ratioAccuracy, ratio, + cSpeedAccuracy, compressionSpeed); + } + } /* if (adv->mode != BMK_DECODE_ONLY) */ + { + BMK_initDCtxArgs dctxprep = { dctx, dictBuffer, dictBufferSize }; + BMK_customReturn_t decompressionResults; - UTIL_sleepMilli(5); /* give processor time to other processes */ - UTIL_waitForNextTick(); + decompressionResults = BMK_benchCustom("ZSTD_decompress_generic", nbBlocks, + (const void * const *)cPtrs, cSizes, resPtrs, resSizes, + &local_initDCtx, &local_defaultDecompress, + (void*)&dctxprep, (void*)(dctx), + adv->loopMode, adv->nbSeconds, displayLevel); - if (!cCompleted) { /* still some time to do compression tests */ - U32 nbLoops = 0; - UTIL_time_t const clockStart = UTIL_getTime(); - ZSTD_CCtx_setParameter(ctx, ZSTD_p_nbWorkers, g_nbWorkers); - ZSTD_CCtx_setParameter(ctx, ZSTD_p_compressionLevel, cLevel); - ZSTD_CCtx_setParameter(ctx, ZSTD_p_enableLongDistanceMatching, g_ldmFlag); - ZSTD_CCtx_setParameter(ctx, ZSTD_p_ldmMinMatch, g_ldmMinMatch); - ZSTD_CCtx_setParameter(ctx, ZSTD_p_ldmHashLog, g_ldmHashLog); - if (g_ldmBucketSizeLog != BMK_LDM_PARAM_NOTSET) { - ZSTD_CCtx_setParameter(ctx, ZSTD_p_ldmBucketSizeLog, g_ldmBucketSizeLog); - } - if (g_ldmHashEveryLog != BMK_LDM_PARAM_NOTSET) { - ZSTD_CCtx_setParameter(ctx, ZSTD_p_ldmHashEveryLog, g_ldmHashEveryLog); - } - ZSTD_CCtx_setParameter(ctx, ZSTD_p_windowLog, comprParams->windowLog); - ZSTD_CCtx_setParameter(ctx, ZSTD_p_hashLog, comprParams->hashLog); - ZSTD_CCtx_setParameter(ctx, ZSTD_p_chainLog, comprParams->chainLog); - ZSTD_CCtx_setParameter(ctx, ZSTD_p_searchLog, comprParams->searchLog); - ZSTD_CCtx_setParameter(ctx, ZSTD_p_minMatch, comprParams->searchLength); - ZSTD_CCtx_setParameter(ctx, ZSTD_p_targetLength, comprParams->targetLength); - ZSTD_CCtx_setParameter(ctx, ZSTD_p_compressionStrategy, comprParams->strategy); - ZSTD_CCtx_loadDictionary(ctx, dictBuffer, dictBufferSize); - - if (!g_nbSeconds) nbCompressionLoops=1; - for (nbLoops=0; nbLoops 0) { - if (loopDuration < fastestC * nbCompressionLoops) - fastestC = loopDuration / nbCompressionLoops; - nbCompressionLoops = (U32)(TIMELOOP_NANOSEC / fastestC) + 1; - } else { - assert(nbCompressionLoops < 40000000); /* avoid overflow */ - nbCompressionLoops *= 100; - } - totalCTime += loopDuration; - cCompleted = (totalCTime >= maxTime); /* end compression tests */ - } } - - cSize = 0; - { U32 blockNb; for (blockNb=0; blockNb%10u (%5.*f),%6.*f MB/s\r", - marks[markNb], displayName, (U32)srcSize, (U32)cSize, - ratioAccuracy, ratio, - cSpeedAccuracy, compressionSpeed ); - } - } /* if (!g_decodeOnly) */ - -#if 0 /* disable decompression test */ - dCompleted=1; - (void)totalDTime; (void)fastestD; (void)crcOrig; /* unused when decompression disabled */ -#else - /* Decompression */ - if (!dCompleted) memset(resultBuffer, 0xD6, srcSize); /* warm result buffer */ - - UTIL_sleepMilli(5); /* give processor time to other processes */ - UTIL_waitForNextTick(); - - if (!dCompleted) { - U32 nbLoops = 0; - ZSTD_DDict* const ddict = ZSTD_createDDict(dictBuffer, dictBufferSize); - UTIL_time_t const clockStart = UTIL_getTime(); - if (!ddict) EXM_THROW(2, "ZSTD_createDDict() allocation failure"); - if (!g_nbSeconds) nbDecodeLoops = 1; - for (nbLoops=0; nbLoops < nbDecodeLoops; nbLoops++) { - U32 blockNb; - for (blockNb=0; blockNb 0) { - if (loopDuration < fastestD * nbDecodeLoops) - fastestD = loopDuration / nbDecodeLoops; - nbDecodeLoops = (U32)(TIMELOOP_NANOSEC / fastestD) + 1; - } else { - assert(nbDecodeLoops < 40000000); /* avoid overflow */ - nbDecodeLoops *= 100; - } - totalDTime += loopDuration; - dCompleted = (totalDTime >= maxTime); - } } + if(decompressionResults.error) { + results.error = decompressionResults.error; + return results; + } markNb = (markNb+1) % NB_MARKS; { int const ratioAccuracy = (ratio < 10.) ? 3 : 2; - double const compressionSpeed = ((double)srcSize / fastestC) * 1000; + double const compressionSpeed = results.result.cSpeed / 1000000; int const cSpeedAccuracy = (compressionSpeed < 10.) ? 2 : 1; - double const decompressionSpeed = ((double)srcSize / fastestD) * 1000; - results.result.cSpeed = compressionSpeed * 1000000; + double const decompressionSpeed = ((double)srcSize / decompressionResults.result.time) * 1000; results.result.dSpeed = decompressionSpeed * 1000000; DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.*f),%6.*f MB/s ,%6.1f MB/s \r", - marks[markNb], displayName, (U32)srcSize, (U32)cSize, + marks[markNb], displayName, (U32)srcSize, (U32)results.result.cSize, ratioAccuracy, ratio, cSpeedAccuracy, compressionSpeed, decompressionSpeed); } - - /* CRC Checking */ - { U64 const crcCheck = XXH64(resultBuffer, srcSize, 0); - if (!g_decodeOnly && (crcOrig!=crcCheck)) { - size_t u; - DISPLAY("!!! WARNING !!! %14s : Invalid Checksum : %x != %x \n", displayName, (unsigned)crcOrig, (unsigned)crcCheck); - for (u=0; u u) break; - bacc += blockTable[segNb].srcSize; - } - pos = (U32)(u - bacc); - bNb = pos / (128 KB); - DISPLAY("(sample %u, block %u, pos %u) \n", segNb, bNb, pos); - if (u>5) { - int n; - DISPLAY("origin: "); - for (n=-5; n<0; n++) DISPLAY("%02X ", ((const BYTE*)srcBuffer)[u+n]); - DISPLAY(" :%02X: ", ((const BYTE*)srcBuffer)[u]); - for (n=1; n<3; n++) DISPLAY("%02X ", ((const BYTE*)srcBuffer)[u+n]); - DISPLAY(" \n"); - DISPLAY("decode: "); - for (n=-5; n<0; n++) DISPLAY("%02X ", ((const BYTE*)resultBuffer)[u+n]); - DISPLAY(" :%02X: ", ((const BYTE*)resultBuffer)[u]); - for (n=1; n<3; n++) DISPLAY("%02X ", ((const BYTE*)resultBuffer)[u+n]); - DISPLAY(" \n"); - } - break; - } - if (u==srcSize-1) { /* should never happen */ - DISPLAY("no difference detected\n"); - } } - break; - } } /* CRC Checking */ -#endif - } /* for (testNb = 1; testNb <= (g_nbSeconds + !g_nbSeconds); testNb++) */ - - if (displayLevel == 1) { /* hidden display mode -q, used by python speed benchmark */ - double const cSpeed = ((double)srcSize / fastestC) * 1000; - double const dSpeed = ((double)srcSize / fastestD) * 1000; - if (g_additionalParam) - DISPLAY("-%-3i%11i (%5.3f) %6.2f MB/s %6.1f MB/s %s (param=%d)\n", cLevel, (int)cSize, ratio, cSpeed, dSpeed, displayName, g_additionalParam); - else - DISPLAY("-%-3i%11i (%5.3f) %6.2f MB/s %6.1f MB/s %s\n", cLevel, (int)cSize, ratio, cSpeed, dSpeed, displayName); } - DISPLAYLEVEL(2, "%2i#\n", cLevel); - } /* Bench */ + + /* CRC Checking */ + { U64 const crcCheck = XXH64(resultBuffer, srcSize, 0); + if ((adv->mode != BMK_DECODE_ONLY) && (crcOrig!=crcCheck)) { + size_t u; + DISPLAY("!!! WARNING !!! %14s : Invalid Checksum : %x != %x \n", displayName, (unsigned)crcOrig, (unsigned)crcCheck); + for (u=0; u u) break; + bacc += srcSizes[segNb]; + } + pos = (U32)(u - bacc); + bNb = pos / (128 KB); + DISPLAY("(sample %u, block %u, pos %u) \n", segNb, bNb, pos); + if (u>5) { + int n; + DISPLAY("origin: "); + for (n=-5; n<0; n++) DISPLAY("%02X ", ((const BYTE*)srcBuffer)[u+n]); + DISPLAY(" :%02X: ", ((const BYTE*)srcBuffer)[u]); + for (n=1; n<3; n++) DISPLAY("%02X ", ((const BYTE*)srcBuffer)[u+n]); + DISPLAY(" \n"); + DISPLAY("decode: "); + for (n=-5; n<0; n++) DISPLAY("%02X ", ((const BYTE*)resultBuffer)[u+n]); + DISPLAY(" :%02X: ", ((const BYTE*)resultBuffer)[u]); + for (n=1; n<3; n++) DISPLAY("%02X ", ((const BYTE*)resultBuffer)[u+n]); + DISPLAY(" \n"); + } + break; + } + if (u==srcSize-1) { /* should never happen */ + DISPLAY("no difference detected\n"); + } + } + } + } /* CRC Checking */ + + if (displayLevel == 1) { /* hidden display mode -q, used by python speed benchmark */ + double const cSpeed = results.result.cSpeed / 1000000; + double const dSpeed = results.result.dSpeed / 1000000; + if (adv->additionalParam) + DISPLAY("-%-3i%11i (%5.3f) %6.2f MB/s %6.1f MB/s %s (param=%d)\n", cLevel, (int)cSize, ratio, cSpeed, dSpeed, displayName, adv->additionalParam); + else + DISPLAY("-%-3i%11i (%5.3f) %6.2f MB/s %6.1f MB/s %s\n", cLevel, (int)cSize, ratio, cSpeed, dSpeed, displayName); + } + DISPLAYLEVEL(2, "%2i#\n", cLevel); +} /* Bench */ /* clean up */ - free(blockTable); free(compressedBuffer); free(resultBuffer); - results.errorCode = 0; + + free(srcPtrs); + free(srcSizes); + free(cPtrs); + free(cSizes); + free(resPtrs); + free(resSizes); + + results.error = 0; return results; } -static void BMK_benchMemCtxless(const void* srcBuffer, size_t srcSize, +BMK_return_t BMK_benchMem(const void* srcBuffer, size_t srcSize, + const size_t* fileSizes, unsigned nbFiles, + const int cLevel, const ZSTD_compressionParameters* comprParams, + const void* dictBuffer, size_t dictBufferSize, + ZSTD_CCtx* ctx, ZSTD_DCtx* dctx, + int displayLevel, const char* displayName) { + + const BMK_advancedParams_t adv = BMK_defaultAdvancedParams(); + return BMK_benchMemAdvanced(srcBuffer, srcSize, + fileSizes, nbFiles, + cLevel, comprParams, + dictBuffer, dictBufferSize, + ctx, dctx, + displayLevel, displayName, &adv); +} + +static BMK_return_t BMK_benchMemCtxless(const void* srcBuffer, size_t srcSize, const size_t* fileSizes, unsigned nbFiles, int cLevel, const ZSTD_compressionParameters* const comprParams, const void* dictBuffer, size_t dictBufferSize, - int displayLevel, const char* displayName) + int displayLevel, const char* displayName, + const BMK_advancedParams_t * const adv) { + BMK_return_t res; ZSTD_CCtx* ctx = ZSTD_createCCtx(); ZSTD_DCtx* dctx = ZSTD_createDCtx(); if(ctx == NULL || dctx == NULL) { - EXM_THROW(12, "not enough memory for contexts"); + EXM_THROW(12, BMK_return_t, "not enough memory for contexts"); } - BMK_benchMem(srcBuffer, srcSize, + res = BMK_benchMemAdvanced(srcBuffer, srcSize, fileSizes, nbFiles, cLevel, comprParams, dictBuffer, dictBufferSize, ctx, dctx, - displayLevel, displayName); + displayLevel, displayName, adv); ZSTD_freeCCtx(ctx); ZSTD_freeDCtx(dctx); + return res; } static size_t BMK_findMaxMem(U64 requiredMem) @@ -544,44 +677,59 @@ static size_t BMK_findMaxMem(U64 requiredMem) return (size_t)(requiredMem); } +ERROR_STRUCT(BMK_result_t*, BMK_returnPtr_t); + /* returns average stats over all range [cLevel, cLevelLast] */ -static void BMK_benchCLevel(const void* srcBuffer, size_t benchedSize, +static BMK_returnPtr_t BMK_benchCLevel(const void* srcBuffer, size_t benchedSize, const size_t* fileSizes, unsigned nbFiles, const int cLevel, const int cLevelLast, const ZSTD_compressionParameters* comprParams, const void* dictBuffer, size_t dictBufferSize, - int displayLevel, const char* displayName) + int displayLevel, const char* displayName, + BMK_advancedParams_t const * const adv) { int l; + BMK_result_t* res = (BMK_result_t*)malloc(sizeof(BMK_result_t) * (cLevelLast - cLevel + 1)); + BMK_returnPtr_t ret = { 0, res }; const char* pch = strrchr(displayName, '\\'); /* Windows */ if (!pch) pch = strrchr(displayName, '/'); /* Linux */ if (pch) displayName = pch+1; - if (g_realTime) { + if(res == NULL) { + EXM_THROW(12, BMK_returnPtr_t, "not enough memory\n"); + } + if (adv->realTime) { DISPLAYLEVEL(2, "Note : switching to real-time priority \n"); SET_REALTIME_PRIORITY; } - if (displayLevel == 1 && !g_additionalParam) - DISPLAY("bench %s %s: input %u bytes, %u seconds, %u KB blocks\n", ZSTD_VERSION_STRING, ZSTD_GIT_COMMIT_STRING, (U32)benchedSize, g_nbSeconds, (U32)(g_blockSize>>10)); + if (displayLevel == 1 && !adv->additionalParam) + DISPLAY("bench %s %s: input %u bytes, %u seconds, %u KB blocks\n", ZSTD_VERSION_STRING, ZSTD_GIT_COMMIT_STRING, (U32)benchedSize, adv->nbSeconds, (U32)(adv->blockSize>>10)); for (l=cLevel; l <= cLevelLast; l++) { + BMK_return_t rettmp; if (l==0) continue; /* skip level 0 */ - BMK_benchMemCtxless(srcBuffer, benchedSize, - fileSizes, nbFiles, - l, comprParams, - dictBuffer, dictBufferSize, - displayLevel, displayName); + rettmp = BMK_benchMemCtxless(srcBuffer, benchedSize, + fileSizes, nbFiles, + l, comprParams, + dictBuffer, dictBufferSize, + displayLevel, displayName, + adv); + if(rettmp.error) { + ret.error = rettmp.error; + return ret; + } + res[l-cLevel] = rettmp.result; } - return; + return ret; } /*! BMK_loadFiles() : * Loads `buffer` with content of files listed within `fileNamesTable`. * At most, fills `buffer` entirely. */ -static void BMK_loadFiles(void* buffer, size_t bufferSize, +static int BMK_loadFiles(void* buffer, size_t bufferSize, size_t* fileSizes, const char* const * const fileNamesTable, unsigned nbFiles, int displayLevel) { @@ -601,44 +749,55 @@ static void BMK_loadFiles(void* buffer, size_t bufferSize, continue; } f = fopen(fileNamesTable[n], "rb"); - if (f==NULL) EXM_THROW(10, "impossible to open file %s", fileNamesTable[n]); + if (f==NULL) EXM_THROW_INT(10, "impossible to open file %s", fileNamesTable[n]); DISPLAYUPDATE(2, "Loading %s... \r", fileNamesTable[n]); if (fileSize > bufferSize-pos) fileSize = bufferSize-pos, nbFiles=n; /* buffer too small - stop after this file */ { size_t const readSize = fread(((char*)buffer)+pos, 1, (size_t)fileSize, f); - if (readSize != (size_t)fileSize) EXM_THROW(11, "could not read %s", fileNamesTable[n]); + if (readSize != (size_t)fileSize) EXM_THROW_INT(11, "could not read %s", fileNamesTable[n]); pos += readSize; } fileSizes[n] = (size_t)fileSize; totalSize += (size_t)fileSize; fclose(f); } - if (totalSize == 0) EXM_THROW(12, "no data to bench"); + if (totalSize == 0) EXM_THROW_INT(12, "no data to bench"); + return 0; } -static void BMK_benchFileTable(const char* const * const fileNamesTable, unsigned const nbFiles, +static BMK_returnSet_t BMK_benchFileTable(const char* const * const fileNamesTable, unsigned const nbFiles, const char* const dictFileName, int const cLevel, int const cLevelLast, - const ZSTD_compressionParameters* const compressionParams, int displayLevel) + const ZSTD_compressionParameters* const compressionParams, int displayLevel, + const BMK_advancedParams_t * const adv) { void* srcBuffer; size_t benchedSize; void* dictBuffer = NULL; size_t dictBufferSize = 0; size_t* const fileSizes = (size_t*)malloc(nbFiles * sizeof(size_t)); + BMK_returnSet_t res; U64 const totalSizeToLoad = UTIL_getTotalFileSize(fileNamesTable, nbFiles); - if (!fileSizes) EXM_THROW(12, "not enough memory for fileSizes"); + res.result.cLevel = cLevel; + res.result.cLevelLast = cLevelLast; + if (!fileSizes) EXM_THROW(12, BMK_returnSet_t, "not enough memory for fileSizes"); /* Load dictionary */ if (dictFileName != NULL) { U64 const dictFileSize = UTIL_getFileSize(dictFileName); if (dictFileSize > 64 MB) - EXM_THROW(10, "dictionary file %s too large", dictFileName); + EXM_THROW(10, BMK_returnSet_t, "dictionary file %s too large", dictFileName); dictBufferSize = (size_t)dictFileSize; dictBuffer = malloc(dictBufferSize); if (dictBuffer==NULL) - EXM_THROW(11, "not enough memory for dictionary (%u bytes)", + EXM_THROW(11, BMK_returnSet_t, "not enough memory for dictionary (%u bytes)", (U32)dictBufferSize); - BMK_loadFiles(dictBuffer, dictBufferSize, fileSizes, &dictFileName, 1, displayLevel); + { + int errorCode = BMK_loadFiles(dictBuffer, dictBufferSize, fileSizes, &dictFileName, 1, displayLevel); + if(errorCode) { + res.error = errorCode; + return res; + } + } } /* Memory allocation & restrictions */ @@ -647,76 +806,112 @@ static void BMK_benchFileTable(const char* const * const fileNamesTable, unsigne if (benchedSize < totalSizeToLoad) DISPLAY("Not enough memory; testing %u MB only...\n", (U32)(benchedSize >> 20)); srcBuffer = malloc(benchedSize); - if (!srcBuffer) EXM_THROW(12, "not enough memory"); + if (!srcBuffer) EXM_THROW(12, BMK_returnSet_t, "not enough memory"); /* Load input buffer */ - BMK_loadFiles(srcBuffer, benchedSize, fileSizes, fileNamesTable, nbFiles, displayLevel); - + { + int errorCode = BMK_loadFiles(srcBuffer, benchedSize, fileSizes, fileNamesTable, nbFiles, displayLevel); + if(errorCode) { + res.error = errorCode; + return res; + } + } /* Bench */ - if (g_separateFiles) { + if (adv->separateFiles) { const BYTE* srcPtr = (const BYTE*)srcBuffer; U32 fileNb; - BMK_result_t* resultarray = (BMK_result_t*)malloc(sizeof(BMK_result_t) * nbFiles); - if(resultarray == NULL) EXM_THROW(12, "not enough memory"); + res.result.results = (BMK_result_t**)malloc(sizeof(BMK_result_t*) * nbFiles); + res.result.nbFiles = nbFiles; + if(res.result.results == NULL) EXM_THROW(12, BMK_returnSet_t, "not enough memory"); for (fileNb=0; fileNb 1) ? mfName : fileNamesTable[0]; - BMK_benchCLevel(srcBuffer, benchedSize, + { + const char* const displayName = (nbFiles > 1) ? mfName : fileNamesTable[0]; + res.result.results = (BMK_result_t**)malloc(sizeof(BMK_result_t*)); + BMK_returnPtr_t errorOrPtr = BMK_benchCLevel(srcBuffer, benchedSize, fileSizes, nbFiles, cLevel, cLevelLast, compressionParams, dictBuffer, dictBufferSize, - displayLevel, displayName); + displayLevel, displayName, + adv); + if(res.result.results == NULL) EXM_THROW(12, BMK_returnSet_t, "not enough memory"); + if(errorOrPtr.error) { + res.error = errorOrPtr.error; + return res; + } + res.result.results[0] = errorOrPtr.result; } } /* clean up */ free(srcBuffer); free(dictBuffer); free(fileSizes); + res.error = 0; + return res; } -static void BMK_syntheticTest(int cLevel, int cLevelLast, double compressibility, +static BMK_returnSet_t BMK_syntheticTest(int cLevel, int cLevelLast, double compressibility, const ZSTD_compressionParameters* compressionParams, - int displayLevel) + int displayLevel, const BMK_advancedParams_t * const adv) { char name[20] = {0}; size_t benchedSize = 10000000; void* const srcBuffer = malloc(benchedSize); - + BMK_returnSet_t res; + res.result.results = malloc(sizeof(BMK_result_t*)); + res.result.nbFiles = 1; + res.result.cLevel = cLevel; + res.result.cLevelLast = cLevelLast; /* Memory allocation */ - if (!srcBuffer) EXM_THROW(21, "not enough memory"); + if (!srcBuffer || !res.result.results) EXM_THROW(21, BMK_returnSet_t, "not enough memory"); /* Fill input buffer */ RDG_genBuffer(srcBuffer, benchedSize, compressibility, 0.0, 0); /* Bench */ snprintf (name, sizeof(name), "Synthetic %2u%%", (unsigned)(compressibility*100)); - BMK_benchCLevel(srcBuffer, benchedSize, + BMK_returnPtr_t errPtr = BMK_benchCLevel(srcBuffer, benchedSize, &benchedSize, 1, cLevel, cLevelLast, compressionParams, NULL, 0, - displayLevel, name); + displayLevel, name, adv); + if(errPtr.error) { + res.error = errPtr.error; + return res; + } + res.result.results[0] = errPtr.result; /* clean up */ free(srcBuffer); + res.error = 0; + return res; } -static void BMK_benchFilesFull(const char** fileNamesTable, unsigned nbFiles, +BMK_returnSet_t BMK_benchFilesAdvanced(const char** fileNamesTable, unsigned nbFiles, const char* dictFileName, int cLevel, int cLevelLast, - const ZSTD_compressionParameters* compressionParams, int displayLevel) + const ZSTD_compressionParameters* compressionParams, + int displayLevel, const BMK_advancedParams_t * const adv) { double const compressibility = (double)g_compressibilityDefault / 100; @@ -726,10 +921,12 @@ static void BMK_benchFilesFull(const char** fileNamesTable, unsigned nbFiles, if (cLevelLast > cLevel) DISPLAYLEVEL(2, "Benchmarking levels from %d to %d\n", cLevel, cLevelLast); - if (nbFiles == 0) - BMK_syntheticTest(cLevel, cLevelLast, compressibility, compressionParams, displayLevel); - else - BMK_benchFileTable(fileNamesTable, nbFiles, dictFileName, cLevel, cLevelLast, compressionParams, displayLevel); + if (nbFiles == 0) { + return BMK_syntheticTest(cLevel, cLevelLast, compressibility, compressionParams, displayLevel, adv); + } + else { + return BMK_benchFileTable(fileNamesTable, nbFiles, dictFileName, cLevel, cLevelLast, compressionParams, displayLevel, adv); + } } int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles, @@ -737,6 +934,21 @@ int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles, int cLevel, int cLevelLast, const ZSTD_compressionParameters* compressionParams, int displayLevel) { - BMK_benchFilesFull(fileNamesTable, nbFiles, dictFileName, cLevel, cLevelLast, compressionParams, displayLevel); - return 0; + const BMK_advancedParams_t adv = BMK_defaultAdvancedParams(); + return BMK_benchFilesAdvanced(fileNamesTable, nbFiles, dictFileName, cLevel, cLevelLast, compressionParams, displayLevel, &adv).error; +} + +/* errorable or just return? */ +BMK_result_t BMK_getResult(BMK_resultSet_t resultSet, unsigned fileIdx, int cLevel) { + assert(resultSet.nbFiles > fileIdx); + assert(resultSet.cLevel <= cLevel && cLevel <= resultSet.cLevelLast); + return resultSet.results[fileIdx][cLevel - resultSet.cLevel]; +} + +void BMK_freeResultSet(BMK_resultSet_t src) { + unsigned i; + for(i = 0; i <= src.nbFiles; i++) { + free(src.results[i]); + } + free(src.results); } diff --git a/programs/bench.h b/programs/bench.h index 0ba6f8985..ad2682e9a 100644 --- a/programs/bench.h +++ b/programs/bench.h @@ -19,25 +19,97 @@ extern "C" { #define ZSTD_STATIC_LINKING_ONLY /* ZSTD_compressionParameters */ #include "zstd.h" /* ZSTD_compressionParameters */ +#define BMK_COMPRESS_ONLY 2 +#define BMK_DECODE_ONLY 1 + +#define TIME_MODE = 0 +#define ITER_MODE = 1 + +#define ERROR_STRUCT(baseType, typeName) typedef struct { \ + int error; \ + baseType result; \ +} typeName + typedef struct { size_t cSize; double cSpeed; /* bytes / sec */ double dSpeed; } BMK_result_t; -/* 0 = no Error */ typedef struct { - int errorCode; - BMK_result_t result; -} BMK_return_t; + int cLevel; + int cLevelLast; + unsigned nbFiles; + BMK_result_t** results; +} BMK_resultSet_t; -/* called in cli */ -int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles, const char* dictFileName, - int cLevel, int cLevelLast, const ZSTD_compressionParameters* compressionParams, - int displayLevel); +typedef struct { + size_t size; + U64 time; +} BMK_customResult_t; -/* basic benchmarking function, called in paramgrill - * ctx, dctx must be valid */ + +ERROR_STRUCT(BMK_result_t, BMK_return_t); +ERROR_STRUCT(BMK_resultSet_t, BMK_returnSet_t); +ERROR_STRUCT(BMK_customResult_t, BMK_customReturn_t); + +/* want all 0 to be default, but wb ldmBucketSizeLog/ldmHashEveryLog */ +typedef struct { + unsigned mode; /* 0: all, 1: compress only 2: decode only */ + int loopMode; /* if loopmode, then nbSeconds = nbLoops */ + unsigned nbSeconds; /* default timing is in nbSeconds. If nbCycles != 0 then use that */ + size_t blockSize; /* Maximum allowable size of a block*/ + unsigned nbWorkers; /* multithreading */ + unsigned realTime; + unsigned separateFiles; + int additionalParam; + unsigned ldmFlag; + unsigned ldmMinMatch; + unsigned ldmHashLog; + unsigned ldmBucketSizeLog; + unsigned ldmHashEveryLog; +} BMK_advancedParams_t; + +/* returns default parameters used by nonAdvanced functions */ +BMK_advancedParams_t BMK_defaultAdvancedParams(void); + +/* functionName - name of function + * blockCount - number of blocks (size of srcBuffers, srcSizes, dstBuffers, dstSizes) + * initFn - (*initFn)(initPayload) is run once per benchmark + * benchFn - (*benchFn)(srcBuffers[i], srcSizes[i], dstBuffers[i], dstSizes[i], benchPayload) + * is run a variable number of times, specified by mode and iter args + * mode - if 0, iter will be interpreted as the minimum number of seconds to run + * iter - see mode + * displayLevel - what gets printed + * 0 : no display; + * 1 : errors; + * 2 : + result + interaction + warnings; + * 3 : + progression; + * 4 : + information + * return + * .error will give a nonzero value if any error has occured + * .result will contain the speed (B/s) and time per loop (ns) + */ +BMK_customReturn_t BMK_benchCustom(const char* functionName, size_t blockCount, + const void* const * const srcBuffers, size_t* srcSizes, + void* const * const dstBuffers, size_t* dstSizes, + size_t (*initFn)(void*), size_t (*benchFn)(const void*, size_t, void*, size_t, void*), + void* initPayload, void* benchPayload, + unsigned mode, unsigned iter, + int displayLevel); + +/* basic benchmarking function, called in paramgrill ctx, dctx must be provided */ +/* srcBuffer - data source, expected to be valid compressed data if in Decode Only Mode + * srcSize - size of data in srcBuffer + * cLevel - compression level + * comprParams - basic compression parameters + * dictBuffer - a dictionary if used, null otherwise + * dictBufferSize - size of dictBuffer, 0 otherwise + * ctx - Compression Context + * dctx - Decompression Context + * diplayLevel - see BMK_benchCustom + * displayName - name used in display + */ BMK_return_t BMK_benchMem(const void* srcBuffer, size_t srcSize, const size_t* fileSizes, unsigned nbFiles, const int cLevel, const ZSTD_compressionParameters* comprParams, @@ -45,20 +117,37 @@ BMK_return_t BMK_benchMem(const void* srcBuffer, size_t srcSize, ZSTD_CCtx* ctx, ZSTD_DCtx* dctx, int displayLevel, const char* displayName); -/* Set Parameters */ -void BMK_setNbSeconds(unsigned nbLoops); -void BMK_setBlockSize(size_t blockSize); -void BMK_setNbWorkers(unsigned nbWorkers); -void BMK_setRealTime(unsigned priority); -void BMK_setNotificationLevel(unsigned level); -void BMK_setSeparateFiles(unsigned separate); -void BMK_setAdditionalParam(int additionalParam); -void BMK_setDecodeOnlyMode(unsigned decodeFlag); -void BMK_setLdmFlag(unsigned ldmFlag); -void BMK_setLdmMinMatch(unsigned ldmMinMatch); -void BMK_setLdmHashLog(unsigned ldmHashLog); -void BMK_setLdmBucketSizeLog(unsigned ldmBucketSizeLog); -void BMK_setLdmHashEveryLog(unsigned ldmHashEveryLog); +BMK_return_t BMK_benchMemAdvanced(const void* srcBuffer, size_t srcSize, + const size_t* fileSizes, unsigned nbFiles, + const int cLevel, const ZSTD_compressionParameters* comprParams, + const void* dictBuffer, size_t dictBufferSize, + ZSTD_CCtx* ctx, ZSTD_DCtx* dctx, + int displayLevel, const char* displayName, + const BMK_advancedParams_t* adv); + +/* called in cli */ +/* fileNamesTable - name of files to benchmark + * nbFiles - number of files (size of fileNamesTable) + * dictFileName - name of dictionary file to load + * cLevel - lowest compression level to benchmark + * cLevellast - highest compression level to benchmark (everything in the range [cLevel, cLevellast]) will be benchmarked + * compressionParams - basic compression Parameters + * displayLevel - see BMK_benchCustom + */ +int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles, const char* dictFileName, + int cLevel, int cLevelLast, const ZSTD_compressionParameters* compressionParams, + int displayLevel); + +BMK_returnSet_t BMK_benchFilesAdvanced(const char** fileNamesTable, unsigned nbFiles, + const char* dictFileName, + int cLevel, int cLevelLast, + const ZSTD_compressionParameters* compressionParams, + int displayLevel, const BMK_advancedParams_t* adv); + +/* get data from resultSet */ +/* when aggregated (separateFiles = 0), just be getResult(r,0,cl) */ +BMK_result_t BMK_getResult(BMK_resultSet_t results, unsigned fileIdx, int cLevel); +void BMK_freeResultSet(BMK_resultSet_t src); #endif /* BENCH_H_121279284357 */ diff --git a/programs/zstdcli.c b/programs/zstdcli.c index 6b6a93528..73fb52169 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -398,6 +398,7 @@ int main(int argCount, const char* argv[]) setRealTimePrio = 0, singleThread = 0, ultra=0; + BMK_advancedParams_t adv = BMK_defaultAdvancedParams(); unsigned bench_nbSeconds = 3; /* would be better if this value was synchronized from bench */ size_t blockSize = 0; zstd_operation_mode operation = zom_compress; @@ -607,7 +608,7 @@ int main(int argCount, const char* argv[]) /* Decoding */ case 'd': #ifndef ZSTD_NOBENCH - BMK_setDecodeOnlyMode(1); + adv.mode = BMK_DECODE_ONLY; if (operation==zom_bench) { argument++; break; } /* benchmark decode (hidden option) */ #endif operation=zom_decompress; argument++; break; @@ -700,7 +701,7 @@ int main(int argCount, const char* argv[]) case 'p': argument++; #ifndef ZSTD_NOBENCH if ((*argument>='0') && (*argument<='9')) { - BMK_setAdditionalParam(readU32FromChar(&argument)); + adv.additionalParam = (int)readU32FromChar(&argument); } else #endif main_pause=1; @@ -801,21 +802,21 @@ int main(int argCount, const char* argv[]) /* Check if benchmark is selected */ if (operation==zom_bench) { #ifndef ZSTD_NOBENCH - BMK_setSeparateFiles(separateFiles); - BMK_setBlockSize(blockSize); - BMK_setNbWorkers(nbWorkers); - BMK_setRealTime(setRealTimePrio); - BMK_setNbSeconds(bench_nbSeconds); - BMK_setLdmFlag(ldmFlag); - BMK_setLdmMinMatch(g_ldmMinMatch); - BMK_setLdmHashLog(g_ldmHashLog); + adv.separateFiles = separateFiles; + adv.blockSize = blockSize; + adv.nbWorkers = nbWorkers; + adv.realTime = setRealTimePrio; + adv.nbSeconds = bench_nbSeconds; + adv.ldmFlag = ldmFlag; + adv.ldmMinMatch = g_ldmMinMatch; + adv.ldmHashLog = g_ldmHashLog; if (g_ldmBucketSizeLog != LDM_PARAM_DEFAULT) { - BMK_setLdmBucketSizeLog(g_ldmBucketSizeLog); + adv.ldmBucketSizeLog = g_ldmBucketSizeLog; } if (g_ldmHashEveryLog != LDM_PARAM_DEFAULT) { - BMK_setLdmHashEveryLog(g_ldmHashEveryLog); + adv.ldmHashEveryLog = g_ldmHashEveryLog; } - BMK_benchFiles(filenameTable, filenameIdx, dictFileName, cLevel, cLevelLast, &compressionParams, g_displayLevel); + BMK_benchFilesAdvanced(filenameTable, filenameIdx, dictFileName, cLevel, cLevelLast, &compressionParams, g_displayLevel, &adv); #else (void)bench_nbSeconds; (void)blockSize; (void)setRealTimePrio; (void)separateFiles; #endif diff --git a/tests/paramgrill.c b/tests/paramgrill.c index 2d7e52a43..025bc6aad 100644 --- a/tests/paramgrill.c +++ b/tests/paramgrill.c @@ -162,8 +162,6 @@ const char* g_stratName[ZSTD_btultra+1] = { "ZSTD_btlazy2 ", "ZSTD_btopt ", "ZSTD_btultra "}; /* TODO: support additional parameters (more files, fileSizes) */ - -//TODO: benchMem dctx can't = NULL in new system static size_t BMK_benchParam(BMK_result_t* resultPtr, const void* srcBuffer, size_t srcSize, @@ -172,7 +170,7 @@ BMK_benchParam(BMK_result_t* resultPtr, BMK_return_t res = BMK_benchMem(srcBuffer,srcSize, &srcSize, 1, 0, &cParams, NULL, 0, ctx, dctx, 0, "File"); *resultPtr = res.result; - return res.errorCode; + return res.error; } static void BMK_printWinner(FILE* f, U32 cLevel, BMK_result_t result, ZSTD_compressionParameters params, size_t srcSize) From 8522346322d3209a8d9e078f927028c2e3d35341 Mon Sep 17 00:00:00 2001 From: George Lu Date: Thu, 14 Jun 2018 14:46:17 -0400 Subject: [PATCH 003/372] Make Fullbench use new function Rearrange Args Add nothing function Use new function, change locals to match New Display Comment cleanup Change builds --- .../fullbench-dll/fullbench-dll.vcxproj | 2 + build/VS2010/fullbench/fullbench.vcxproj | 2 + programs/bench.c | 214 ++++++++++-------- programs/bench.h | 21 +- programs/zstdcli.c | 4 +- tests/Makefile | 2 +- tests/fullbench.c | 82 +++---- 7 files changed, 167 insertions(+), 160 deletions(-) diff --git a/build/VS2010/fullbench-dll/fullbench-dll.vcxproj b/build/VS2010/fullbench-dll/fullbench-dll.vcxproj index e697318e0..6939d4406 100644 --- a/build/VS2010/fullbench-dll/fullbench-dll.vcxproj +++ b/build/VS2010/fullbench-dll/fullbench-dll.vcxproj @@ -167,11 +167,13 @@ + + diff --git a/build/VS2010/fullbench/fullbench.vcxproj b/build/VS2010/fullbench/fullbench.vcxproj index d0cbcae98..25ae07d4b 100644 --- a/build/VS2010/fullbench/fullbench.vcxproj +++ b/build/VS2010/fullbench/fullbench.vcxproj @@ -174,6 +174,7 @@ + @@ -195,6 +196,7 @@ + diff --git a/programs/bench.c b/programs/bench.c index 7b9ea8218..ff0403ab1 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -64,8 +64,7 @@ static const size_t maxMemory = (sizeof(size_t)==4) ? (2 GB - 64 MB) : (size_t)(1ULL << ((sizeof(size_t)*8)-31)); -//TODO: remove this gv as well -//Only used in Synthetic test. Separate? +/* remove this in the future? */ static U32 g_compressibilityDefault = 50; /* ************************************* @@ -119,8 +118,8 @@ static UTIL_time_t g_displayClock = UTIL_TIME_INITIALIZER; BMK_advancedParams_t BMK_defaultAdvancedParams(void) { BMK_advancedParams_t res = { - 0, /* mode */ - 0, /* nbCycles */ + BMK_both, /* mode */ + BMK_timeMode, /* loopMode */ BMK_TIMETEST_DEFAULT_S, /* nbSeconds */ 0, /* blockSize */ 0, /* nbWorkers */ @@ -266,18 +265,14 @@ static size_t local_defaultDecompress( } -//ignore above for error stuff, return type still undecided - /* mode 0 : iter = # seconds, else iter = # cycles */ /* initFn will be measured once, bench fn will be measured x times */ /* benchFn should return error value or out Size */ -//problem : how to get cSize this way for ratio? -//also possible fastest rounds down to 0 if 0 < loopDuration < nbLoops (that would mean <1ns / op though) /* takes # of blocks and list of size & stuff for each. */ BMK_customReturn_t BMK_benchCustom( const char* functionName, size_t blockCount, - const void* const * const srcBuffers, size_t* srcSizes, - void* const * const dstBuffers, size_t* dstSizes, + const void* const * const srcBuffers, const size_t* srcSizes, + void* const * const dstBuffers, const size_t* dstSizes, size_t (*initFn)(void*), size_t (*benchFn)(const void*, size_t, void*, size_t, void*), void* initPayload, void* benchPayload, unsigned mode, unsigned iter, @@ -302,9 +297,7 @@ BMK_customReturn_t BMK_benchCustom( /* display last 17 char's of functionName*/ if (strlen(functionName)>17) functionName += strlen(functionName)-17; if(!iter) { - if(mode) { - EXM_THROW(1, BMK_customReturn_t, "nbSeconds must be nonzero \n"); - } else { + if(mode == BMK_iterMode) { EXM_THROW(1, BMK_customReturn_t, "nbLoops must be nonzero \n"); } @@ -314,73 +307,81 @@ BMK_customReturn_t BMK_benchCustom( srcSize += srcSizes[ind]; } - //change to switch if more modes? - if(!mode) { - int completed = 0; - U64 const maxTime = (iter * TIMELOOP_NANOSEC) + 1; - unsigned nbLoops = 1; - UTIL_time_t coolTime = UTIL_getTime(); - while(!completed) { + switch(mode) { + case BMK_timeMode: + { + int completed = 0; + U64 const maxTime = (iter * TIMELOOP_NANOSEC) + 1; + unsigned nbLoops = 1; + UTIL_time_t coolTime = UTIL_getTime(); + while(!completed) { + unsigned i, j; + /* Overheat protection */ + if (UTIL_clockSpanMicro(coolTime) > ACTIVEPERIOD_MICROSEC) { + DISPLAYLEVEL(2, "\rcooling down ... \r"); + UTIL_sleep(COOLPERIOD_SEC); + coolTime = UTIL_getTime(); + } + + for(i = 0; i < blockCount; i++) { + memset(dstBuffers[i], 0xD6, dstSizes[i]); /* warm up and erase result buffer */ + } + + clockStart = UTIL_getTime(); + (*initFn)(initPayload); + + for(i = 0; i < nbLoops; i++) { + for(j = 0; j < blockCount; j++) { + size_t res = (*benchFn)(srcBuffers[j], srcSizes[j], dstBuffers[j], dstSizes[j], benchPayload); + if(ZSTD_isError(res)) { + EXM_THROW(2, BMK_customReturn_t, "%s() failed on block %u of size %u : %s \n", + functionName, j, (U32)dstSizes[j], ZSTD_getErrorName(res)); + } else if (toAdd) { + dstSize += res; + } + } + toAdd = 0; + } + { U64 const loopDuration = UTIL_clockSpanNano(clockStart); + if (loopDuration > 0) { + fastest = MIN(fastest, loopDuration / nbLoops); + nbLoops = (U32)(TIMELOOP_NANOSEC / fastest) + 1; + } else { + assert(nbLoops < 40000000); /* avoid overflow */ + nbLoops *= 100; + } + totalTime += loopDuration; + completed = (totalTime >= maxTime); + } + } + break; + } + case BMK_iterMode: + { unsigned i, j; - /* Overheat protection */ - if (UTIL_clockSpanMicro(coolTime) > ACTIVEPERIOD_MICROSEC) { - DISPLAYLEVEL(2, "\rcooling down ... \r"); - UTIL_sleep(COOLPERIOD_SEC); - coolTime = UTIL_getTime(); - } - - for(i = 0; i < blockCount; i++) { - memset(dstBuffers[i], 0xD6, dstSizes[i]); /* warm up and erase result buffer */ - } - clockStart = UTIL_getTime(); - (*initFn)(initPayload); - - for(i = 0; i < nbLoops; i++) { + for(i = 0; i < iter; i++) { for(j = 0; j < blockCount; j++) { size_t res = (*benchFn)(srcBuffers[j], srcSizes[j], dstBuffers[j], dstSizes[j], benchPayload); if(ZSTD_isError(res)) { EXM_THROW(2, BMK_customReturn_t, "%s() failed on block %u of size %u : %s \n", functionName, j, (U32)dstSizes[j], ZSTD_getErrorName(res)); - } else if (toAdd) { + } else if(toAdd) { dstSize += res; } } toAdd = 0; } - { U64 const loopDuration = UTIL_clockSpanNano(clockStart); - if (loopDuration > 0) { - fastest = MIN(fastest, loopDuration / nbLoops); - nbLoops = (U32)(TIMELOOP_NANOSEC / fastest) + 1; - } else { - assert(nbLoops < 40000000); /* avoid overflow */ - nbLoops *= 100; - } - totalTime += loopDuration; - completed = (totalTime >= maxTime); - } - } - } else { - unsigned i, j; - clockStart = UTIL_getTime(); - for(i = 0; i < iter; i++) { - for(j = 0; j < blockCount; j++) { - size_t res = (*benchFn)(srcBuffers[j], srcSizes[j], dstBuffers[j], dstSizes[j], benchPayload); - if(ZSTD_isError(res)) { - EXM_THROW(2, BMK_customReturn_t, "%s() failed on block %u of size %u : %s \n", - functionName, j, (U32)dstSizes[j], ZSTD_getErrorName(res)); - } else if(toAdd) { - dstSize += res; - } + totalTime = UTIL_clockSpanNano(clockStart); + if(!totalTime) { + EXM_THROW(3, BMK_customReturn_t, "Cycle count (%u) too short to measure \n", iter); + } else { + fastest = totalTime / iter; } - toAdd = 0; - } - totalTime = UTIL_clockSpanNano(clockStart); - if(!totalTime) { - EXM_THROW(3, BMK_customReturn_t, "Cycle count (%u) too short to measure \n", iter); - } else { - fastest = totalTime / iter; + break; } + default: + EXM_THROW(4, BMK_customReturn_t, "Unknown Mode \n"); } retval.error = 0; retval.result.time = fastest; @@ -396,18 +397,18 @@ BMK_return_t BMK_benchMemAdvanced(const void* srcBuffer, size_t srcSize, int displayLevel, const char* displayName, const BMK_advancedParams_t* adv) { - size_t const blockSize = ((adv->blockSize>=32 && (adv->mode != BMK_DECODE_ONLY)) ? adv->blockSize : srcSize) + (!srcSize) /* avoid div by 0 */ ; + size_t const blockSize = ((adv->blockSize>=32 && (adv->mode != BMK_decodeOnly)) ? adv->blockSize : srcSize) + (!srcSize) /* avoid div by 0 */ ; U32 const maxNbBlocks = (U32) ((srcSize + (blockSize-1)) / blockSize) + nbFiles; /* these are the blockTable parameters, just split up */ - const void ** const srcPtrs = malloc(maxNbBlocks * sizeof(void*)); - size_t* const srcSizes = malloc(maxNbBlocks * sizeof(size_t)); + const void ** const srcPtrs = (const void ** const)malloc(maxNbBlocks * sizeof(void*)); + size_t* const srcSizes = (size_t* const)malloc(maxNbBlocks * sizeof(size_t)); - void ** const cPtrs = malloc(maxNbBlocks * sizeof(void*)); - size_t* const cSizes = malloc(maxNbBlocks * sizeof(size_t)); + void ** const cPtrs = (void** const)malloc(maxNbBlocks * sizeof(void*)); + size_t* const cSizes = (size_t* const)malloc(maxNbBlocks * sizeof(size_t)); - void ** const resPtrs = malloc(maxNbBlocks * sizeof(void*)); - size_t* const resSizes = malloc(maxNbBlocks * sizeof(size_t)); + void ** const resPtrs = (void** const)malloc(maxNbBlocks * sizeof(void*)); + size_t* const resSizes = (size_t* const)malloc(maxNbBlocks * sizeof(size_t)); const size_t maxCompressedSize = ZSTD_compressBound(srcSize) + (maxNbBlocks * 1024); /* add some room for safety */ void* compressedBuffer = malloc(maxCompressedSize); @@ -430,7 +431,7 @@ BMK_return_t BMK_benchMemAdvanced(const void* srcBuffer, size_t srcSize, /* init */ if (strlen(displayName)>17) displayName += strlen(displayName)-17; /* display last 17 characters */ - if (adv->mode == BMK_DECODE_ONLY) { /* benchmark only decompression : source must be already compressed */ + if (adv->mode == BMK_decodeOnly) { /* benchmark only decompression : source must be already compressed */ const char* srcPtr = (const char*)srcBuffer; U64 totalDSize64 = 0; U32 fileNb; @@ -458,19 +459,18 @@ BMK_return_t BMK_benchMemAdvanced(const void* srcBuffer, size_t srcSize, U32 fileNb; for (nbBlocks=0, fileNb=0; fileNbmode == BMK_DECODE_ONLY) ? 1 : (U32)((remaining + (blockSize-1)) / blockSize); + U32 const nbBlocksforThisFile = (adv->mode == BMK_decodeOnly) ? 1 : (U32)((remaining + (blockSize-1)) / blockSize); U32 const blockEnd = nbBlocks + nbBlocksforThisFile; for ( ; nbBlocksmode == BMK_DECODE_ONLY) ? thisBlockSize : ZSTD_compressBound(thisBlockSize); - //blockTable[nbBlocks].cSize = blockTable[nbBlocks].cRoom; + cSizes[nbBlocks] = (adv->mode == BMK_decodeOnly) ? thisBlockSize : ZSTD_compressBound(thisBlockSize); resPtrs[nbBlocks] = (void*)resPtr; - resSizes[nbBlocks] = (adv->mode == BMK_DECODE_ONLY) ? (size_t) ZSTD_findDecompressedSize(srcPtr, thisBlockSize) : thisBlockSize; + resSizes[nbBlocks] = (adv->mode == BMK_decodeOnly) ? (size_t) ZSTD_findDecompressedSize(srcPtr, thisBlockSize) : thisBlockSize; srcPtr += thisBlockSize; - cPtr += cSizes[nbBlocks]; //blockTable[nbBlocks].cRoom; + cPtr += cSizes[nbBlocks]; resPtr += thisBlockSize; remaining -= thisBlockSize; } @@ -478,26 +478,29 @@ BMK_return_t BMK_benchMemAdvanced(const void* srcBuffer, size_t srcSize, } /* warmimg up memory */ - if (adv->mode == BMK_DECODE_ONLY) { + if (adv->mode == BMK_decodeOnly) { memcpy(compressedBuffer, srcBuffer, loadedCompressedSize); } else { RDG_genBuffer(compressedBuffer, maxCompressedSize, 0.10, 0.50, 1); } /* Bench */ - - //TODO: Make sure w/o new loop decode_only code isn't run - //TODO: Support nbLoops and nbSeconds { - U64 const crcOrig = (adv->mode == BMK_DECODE_ONLY) ? 0 : XXH64(srcBuffer, srcSize, 0); + U64 const crcOrig = (adv->mode == BMK_decodeOnly) ? 0 : XXH64(srcBuffer, srcSize, 0); # define NB_MARKS 4 const char* const marks[NB_MARKS] = { " |", " /", " =", "\\" }; U32 markNb = 0; DISPLAYLEVEL(2, "\r%79s\r", ""); - if (adv->mode != BMK_DECODE_ONLY) { - BMK_initCCtxArgs cctxprep = { ctx, dictBuffer, dictBufferSize, cLevel, comprParams, adv }; + if (adv->mode != BMK_decodeOnly) { + BMK_initCCtxArgs cctxprep; BMK_customReturn_t compressionResults; + cctxprep.ctx = ctx; + cctxprep.dictBuffer = dictBuffer; + cctxprep.dictBufferSize = dictBufferSize; + cctxprep.cLevel = cLevel; + cctxprep.comprParams = comprParams; + cctxprep.adv = adv; /* Compression */ DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->\r", marks[markNb], displayName, (U32)srcSize); compressionResults = BMK_benchCustom("ZSTD_compress_generic", nbBlocks, @@ -524,11 +527,15 @@ BMK_return_t BMK_benchMemAdvanced(const void* srcBuffer, size_t srcSize, ratioAccuracy, ratio, cSpeedAccuracy, compressionSpeed); } - } /* if (adv->mode != BMK_DECODE_ONLY) */ - { - BMK_initDCtxArgs dctxprep = { dctx, dictBuffer, dictBufferSize }; + } /* if (adv->mode != BMK_decodeOnly) */ + + if(adv->mode != BMK_compressOnly) { + BMK_initDCtxArgs dctxprep; BMK_customReturn_t decompressionResults; + dctxprep.dctx = dctx; + dctxprep.dictBuffer = dictBuffer; + dctxprep.dictBufferSize = dictBufferSize; decompressionResults = BMK_benchCustom("ZSTD_decompress_generic", nbBlocks, (const void * const *)cPtrs, cSizes, resPtrs, resSizes, &local_initDCtx, &local_defaultDecompress, @@ -556,7 +563,8 @@ BMK_return_t BMK_benchMemAdvanced(const void* srcBuffer, size_t srcSize, /* CRC Checking */ { U64 const crcCheck = XXH64(resultBuffer, srcSize, 0); - if ((adv->mode != BMK_DECODE_ONLY) && (crcOrig!=crcCheck)) { + /* adv->mode == 0 -> compress + decompress */ + if ((adv->mode == BMK_both) && (crcOrig!=crcCheck)) { size_t u; DISPLAY("!!! WARNING !!! %14s : Invalid Checksum : %x != %x \n", displayName, (unsigned)crcOrig, (unsigned)crcCheck); for (u=0; u 1) ? mfName : fileNamesTable[0]; res.result.results = (BMK_result_t**)malloc(sizeof(BMK_result_t*)); - BMK_returnPtr_t errorOrPtr = BMK_benchCLevel(srcBuffer, benchedSize, + errorOrPtr = BMK_benchCLevel(srcBuffer, benchedSize, fileSizes, nbFiles, cLevel, cLevelLast, compressionParams, dictBuffer, dictBufferSize, @@ -877,7 +891,8 @@ static BMK_returnSet_t BMK_syntheticTest(int cLevel, int cLevelLast, double comp size_t benchedSize = 10000000; void* const srcBuffer = malloc(benchedSize); BMK_returnSet_t res; - res.result.results = malloc(sizeof(BMK_result_t*)); + BMK_returnPtr_t errPtr; + res.result.results = (BMK_result_t**)calloc(1,sizeof(BMK_result_t*)); res.result.nbFiles = 1; res.result.cLevel = cLevel; res.result.cLevelLast = cLevelLast; @@ -889,7 +904,7 @@ static BMK_returnSet_t BMK_syntheticTest(int cLevel, int cLevelLast, double comp /* Bench */ snprintf (name, sizeof(name), "Synthetic %2u%%", (unsigned)(compressibility*100)); - BMK_returnPtr_t errPtr = BMK_benchCLevel(srcBuffer, benchedSize, + errPtr = BMK_benchCLevel(srcBuffer, benchedSize, &benchedSize, 1, cLevel, cLevelLast, compressionParams, NULL, 0, @@ -901,7 +916,7 @@ static BMK_returnSet_t BMK_syntheticTest(int cLevel, int cLevelLast, double comp res.result.results[0] = errPtr.result; /* clean up */ - free(srcBuffer); + free((void*)srcBuffer); res.error = 0; return res; } @@ -947,7 +962,8 @@ BMK_result_t BMK_getResult(BMK_resultSet_t resultSet, unsigned fileIdx, int cLev void BMK_freeResultSet(BMK_resultSet_t src) { unsigned i; - for(i = 0; i <= src.nbFiles; i++) { + if(src.results == NULL) { return; } + for(i = 0; i < src.nbFiles; i++) { free(src.results[i]); } free(src.results); diff --git a/programs/bench.h b/programs/bench.h index ad2682e9a..67430f33d 100644 --- a/programs/bench.h +++ b/programs/bench.h @@ -19,11 +19,16 @@ extern "C" { #define ZSTD_STATIC_LINKING_ONLY /* ZSTD_compressionParameters */ #include "zstd.h" /* ZSTD_compressionParameters */ -#define BMK_COMPRESS_ONLY 2 -#define BMK_DECODE_ONLY 1 +typedef enum { + BMK_timeMode = 0, + BMK_iterMode = 1 +} BMK_loopMode_t; -#define TIME_MODE = 0 -#define ITER_MODE = 1 +typedef enum { + BMK_both = 0, + BMK_decodeOnly = 1, + BMK_compressOnly = 2 +} BMK_mode_t; #define ERROR_STRUCT(baseType, typeName) typedef struct { \ int error; \ @@ -55,8 +60,8 @@ ERROR_STRUCT(BMK_customResult_t, BMK_customReturn_t); /* want all 0 to be default, but wb ldmBucketSizeLog/ldmHashEveryLog */ typedef struct { - unsigned mode; /* 0: all, 1: compress only 2: decode only */ - int loopMode; /* if loopmode, then nbSeconds = nbLoops */ + BMK_mode_t mode; /* 0: all, 1: compress only 2: decode only */ + BMK_loopMode_t loopMode; /* if loopmode, then nbSeconds = nbLoops */ unsigned nbSeconds; /* default timing is in nbSeconds. If nbCycles != 0 then use that */ size_t blockSize; /* Maximum allowable size of a block*/ unsigned nbWorkers; /* multithreading */ @@ -91,8 +96,8 @@ BMK_advancedParams_t BMK_defaultAdvancedParams(void); * .result will contain the speed (B/s) and time per loop (ns) */ BMK_customReturn_t BMK_benchCustom(const char* functionName, size_t blockCount, - const void* const * const srcBuffers, size_t* srcSizes, - void* const * const dstBuffers, size_t* dstSizes, + const void* const * const srcBuffers, const size_t* srcSizes, + void* const * const dstBuffers, const size_t* dstSizes, size_t (*initFn)(void*), size_t (*benchFn)(const void*, size_t, void*, size_t, void*), void* initPayload, void* benchPayload, unsigned mode, unsigned iter, diff --git a/programs/zstdcli.c b/programs/zstdcli.c index 73fb52169..8b31a98b1 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -608,7 +608,7 @@ int main(int argCount, const char* argv[]) /* Decoding */ case 'd': #ifndef ZSTD_NOBENCH - adv.mode = BMK_DECODE_ONLY; + adv.mode = BMK_decodeOnly; if (operation==zom_bench) { argument++; break; } /* benchmark decode (hidden option) */ #endif operation=zom_decompress; argument++; break; @@ -816,7 +816,7 @@ int main(int argCount, const char* argv[]) if (g_ldmHashEveryLog != LDM_PARAM_DEFAULT) { adv.ldmHashEveryLog = g_ldmHashEveryLog; } - BMK_benchFilesAdvanced(filenameTable, filenameIdx, dictFileName, cLevel, cLevelLast, &compressionParams, g_displayLevel, &adv); + BMK_freeResultSet(BMK_benchFilesAdvanced(filenameTable, filenameIdx, dictFileName, cLevel, cLevelLast, &compressionParams, g_displayLevel, &adv).result); #else (void)bench_nbSeconds; (void)blockSize; (void)setRealTimePrio; (void)separateFiles; #endif diff --git a/tests/Makefile b/tests/Makefile index c1482bc9c..0db60f1b1 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -133,7 +133,7 @@ fullbench fullbench32 : CPPFLAGS += $(MULTITHREAD_CPP) fullbench fullbench32 : LDFLAGS += $(MULTITHREAD_LD) fullbench fullbench32 : DEBUGFLAGS = # turn off assert() for speed measurements fullbench fullbench32 : $(ZSTD_FILES) -fullbench fullbench32 : $(PRGDIR)/datagen.c fullbench.c +fullbench fullbench32 : $(PRGDIR)/datagen.c $(PRGDIR)/bench.c fullbench.c $(CC) $(FLAGS) $^ -o $@$(EXT) fullbench-lib : zstd-staticLib diff --git a/tests/fullbench.c b/tests/fullbench.c index 6abdd4da0..2dee3db94 100644 --- a/tests/fullbench.c +++ b/tests/fullbench.c @@ -30,6 +30,7 @@ #include "zstd.h" /* ZSTD_versionString */ #include "util.h" /* time functions */ #include "datagen.h" +#include "bench.h" /* CustomBench*/ /*_************************************ @@ -93,14 +94,19 @@ static size_t BMK_findMaxMem(U64 requiredMem) /*_******************************************************* * Benchmark wrappers *********************************************************/ -size_t local_ZSTD_compress(void* dst, size_t dstSize, void* buff2, const void* src, size_t srcSize) +size_t local_nothing(void* x) { + (void)x; + return 0; +} + +size_t local_ZSTD_compress(const void* src, size_t srcSize, void* dst, size_t dstSize, void* buff2) { (void)buff2; return ZSTD_compress(dst, dstSize, src, srcSize, 1); } static size_t g_cSize = 0; -size_t local_ZSTD_decompress(void* dst, size_t dstSize, void* buff2, const void* src, size_t srcSize) +size_t local_ZSTD_decompress(const void* src, size_t srcSize, void* dst, size_t dstSize, void* buff2) { (void)src; (void)srcSize; return ZSTD_decompress(dst, dstSize, buff2, g_cSize); @@ -110,14 +116,14 @@ static ZSTD_DCtx* g_zdc = NULL; #ifndef ZSTD_DLL_IMPORT extern size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* ctx, const void* src, size_t srcSize); -size_t local_ZSTD_decodeLiteralsBlock(void* dst, size_t dstSize, void* buff2, const void* src, size_t srcSize) +size_t local_ZSTD_decodeLiteralsBlock(const void* src, size_t srcSize, void* dst, size_t dstSize, void* buff2) { (void)src; (void)srcSize; (void)dst; (void)dstSize; return ZSTD_decodeLiteralsBlock((ZSTD_DCtx*)g_zdc, buff2, g_cSize); } extern size_t ZSTD_decodeSeqHeaders(ZSTD_DCtx* dctx, int* nbSeq, const void* src, size_t srcSize); -size_t local_ZSTD_decodeSeqHeaders(void* dst, size_t dstSize, void* buff2, const void* src, size_t srcSize) +size_t local_ZSTD_decodeSeqHeaders(const void* src, size_t srcSize, void* dst, size_t dstSize, void* buff2) { int nbSeq; (void)src; (void)srcSize; (void)dst; (void)dstSize; @@ -126,7 +132,7 @@ size_t local_ZSTD_decodeSeqHeaders(void* dst, size_t dstSize, void* buff2, const #endif static ZSTD_CStream* g_cstream= NULL; -size_t local_ZSTD_compressStream(void* dst, size_t dstCapacity, void* buff2, const void* src, size_t srcSize) +size_t local_ZSTD_compressStream(const void* src, size_t srcSize, void* dst, size_t dstCapacity, void* buff2) { ZSTD_outBuffer buffOut; ZSTD_inBuffer buffIn; @@ -143,7 +149,7 @@ size_t local_ZSTD_compressStream(void* dst, size_t dstCapacity, void* buff2, con return buffOut.pos; } -static size_t local_ZSTD_compress_generic_end(void* dst, size_t dstCapacity, void* buff2, const void* src, size_t srcSize) +static size_t local_ZSTD_compress_generic_end(const void* src, size_t srcSize, void* dst, size_t dstCapacity, void* buff2) { ZSTD_outBuffer buffOut; ZSTD_inBuffer buffIn; @@ -159,7 +165,7 @@ static size_t local_ZSTD_compress_generic_end(void* dst, size_t dstCapacity, voi return buffOut.pos; } -static size_t local_ZSTD_compress_generic_continue(void* dst, size_t dstCapacity, void* buff2, const void* src, size_t srcSize) +static size_t local_ZSTD_compress_generic_continue(const void* src, size_t srcSize, void* dst, size_t dstCapacity, void* buff2) { ZSTD_outBuffer buffOut; ZSTD_inBuffer buffIn; @@ -176,7 +182,7 @@ static size_t local_ZSTD_compress_generic_continue(void* dst, size_t dstCapacity return buffOut.pos; } -static size_t local_ZSTD_compress_generic_T2_end(void* dst, size_t dstCapacity, void* buff2, const void* src, size_t srcSize) +static size_t local_ZSTD_compress_generic_T2_end(const void* src, size_t srcSize, void* dst, size_t dstCapacity, void* buff2) { ZSTD_outBuffer buffOut; ZSTD_inBuffer buffIn; @@ -193,7 +199,7 @@ static size_t local_ZSTD_compress_generic_T2_end(void* dst, size_t dstCapacity, return buffOut.pos; } -static size_t local_ZSTD_compress_generic_T2_continue(void* dst, size_t dstCapacity, void* buff2, const void* src, size_t srcSize) +static size_t local_ZSTD_compress_generic_T2_continue(const void* src, size_t srcSize, void* dst, size_t dstCapacity, void* buff2) { ZSTD_outBuffer buffOut; ZSTD_inBuffer buffIn; @@ -212,7 +218,7 @@ static size_t local_ZSTD_compress_generic_T2_continue(void* dst, size_t dstCapac } static ZSTD_DStream* g_dstream= NULL; -static size_t local_ZSTD_decompressStream(void* dst, size_t dstCapacity, void* buff2, const void* src, size_t srcSize) +static size_t local_ZSTD_decompressStream(const void* src, size_t srcSize, void* dst, size_t dstCapacity, void* buff2) { ZSTD_outBuffer buffOut; ZSTD_inBuffer buffIn; @@ -231,7 +237,7 @@ static size_t local_ZSTD_decompressStream(void* dst, size_t dstCapacity, void* b static ZSTD_CCtx* g_zcc = NULL; #ifndef ZSTD_DLL_IMPORT -size_t local_ZSTD_compressContinue(void* dst, size_t dstCapacity, void* buff2, const void* src, size_t srcSize) +size_t local_ZSTD_compressContinue(const void* src, size_t srcSize, void* dst, size_t dstCapacity, void* buff2) { (void)buff2; ZSTD_compressBegin(g_zcc, 1 /* compressionLevel */); @@ -239,7 +245,7 @@ size_t local_ZSTD_compressContinue(void* dst, size_t dstCapacity, void* buff2, c } #define FIRST_BLOCK_SIZE 8 -size_t local_ZSTD_compressContinue_extDict(void* dst, size_t dstCapacity, void* buff2, const void* src, size_t srcSize) +size_t local_ZSTD_compressContinue_extDict(const void* src, size_t srcSize, void* dst, size_t dstCapacity, void* buff2) { BYTE firstBlockBuf[FIRST_BLOCK_SIZE]; @@ -255,7 +261,7 @@ size_t local_ZSTD_compressContinue_extDict(void* dst, size_t dstCapacity, void* return ZSTD_compressEnd(g_zcc, dst, dstCapacity, (const BYTE*)src + FIRST_BLOCK_SIZE, srcSize - FIRST_BLOCK_SIZE); } -size_t local_ZSTD_decompressContinue(void* dst, size_t dstCapacity, void* buff2, const void* src, size_t srcSize) +size_t local_ZSTD_decompressContinue(const void* src, size_t srcSize, void* dst, size_t dstCapacity, void* buff2) { size_t regeneratedSize = 0; const BYTE* ip = (const BYTE*)buff2; @@ -288,8 +294,7 @@ static size_t benchMem(const void* src, size_t srcSize, U32 benchNb) size_t const dstBuffSize = ZSTD_compressBound(srcSize); void* buff2; const char* benchName; - size_t (*benchFunction)(void* dst, size_t dstSize, void* verifBuff, const void* src, size_t srcSize); - double bestTime = 100000000.; + size_t (*benchFunction)(const void* src, size_t srcSize, void* dst, size_t dstSize, void* verifBuff); /* Selection */ switch(benchNb) @@ -419,46 +424,23 @@ static size_t benchMem(const void* src, size_t srcSize, U32 benchNb) default : ; } + /* warming up memory */ { size_t i; for (i=0; i %s !! \n", benchName, ZSTD_getErrorName(benchResult)); - exit(1); - } } - { U64 const clockSpanNano = UTIL_clockSpanNano(clockStart); - double const averageTime = (double)clockSpanNano / TIME_SEC_NANOSEC / nbRounds; - if (clockSpanNano > 0) { - if (averageTime < bestTime) bestTime = averageTime; - assert(bestTime > (1./2000000000)); - nbRounds = (U32)(1. / bestTime); /* aim for 1 sec */ - DISPLAY("%2i- %-30.30s : %7.1f MB/s (%9u)\r", - loopNb, benchName, - (double)srcSize / (1 MB) / bestTime, - (U32)benchResult); - } else { - assert(nbRounds < 40000000); /* avoid overflow */ - nbRounds *= 100; - } - } } } - DISPLAY("%2u\n", benchNb); + { + BMK_customReturn_t r = BMK_benchCustom(benchName, 1, &src, &srcSize, (void * const * const)&dstBuff, &dstBuffSize, &local_nothing, benchFunction, + NULL, buff2, BMK_timeMode, 1, 2); + if(r.error) { + DISPLAY("ERROR %d ! ! \n", r.error); + exit(1); + } + DISPLAY("%2u#Speed: %f MB/s - Size: %f MB\n", benchNb, (double)srcSize / r.result.time * 1000, (double)r.result.size / 1000000); + } + _cleanOut: free(dstBuff); free(buff2); From 0d1ee22990a4f989d8991cb3aced85bbaca62d11 Mon Sep 17 00:00:00 2001 From: George Lu Date: Fri, 15 Jun 2018 16:21:08 -0400 Subject: [PATCH 004/372] Requested Changes Add Comment Simplify Interface (Remove resultSet) Reorder Arguments Remove customBench displayLevel Reorder bench.h Change benchFiles return type to match advanced Rename stuff --- programs/bench.c | 231 ++++++++++++++++++--------------------------- programs/bench.h | 150 +++++++++++++++-------------- programs/zstdcli.c | 16 +++- tests/fullbench.c | 8 +- 4 files changed, 188 insertions(+), 217 deletions(-) diff --git a/programs/bench.c b/programs/bench.c index ff0403ab1..6d0e1c1ff 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -110,13 +110,25 @@ static UTIL_time_t g_displayClock = UTIL_TIME_INITIALIZER; return r; \ } +/* error without displaying */ +#define EXM_THROW_ND(errorNum, retType, ...) { \ + retType r; \ + memset(&r, 0, sizeof(retType)); \ + DEBUGOUTPUT("%s: %i: \n", __FILE__, __LINE__); \ + DEBUGOUTPUT("Error %i : ", errorNum); \ + DEBUGOUTPUT(__VA_ARGS__); \ + DEBUGOUTPUT(" \n"); \ + r.error = errorNum; \ + return r; \ +} + /* ************************************* * Benchmark Parameters ***************************************/ #define BMK_LDM_PARAM_NOTSET 9999 -BMK_advancedParams_t BMK_defaultAdvancedParams(void) { +BMK_advancedParams_t BMK_initAdvancedParams(void) { BMK_advancedParams_t res = { BMK_both, /* mode */ BMK_timeMode, /* loopMode */ @@ -124,7 +136,6 @@ BMK_advancedParams_t BMK_defaultAdvancedParams(void) { 0, /* blockSize */ 0, /* nbWorkers */ 0, /* realTime */ - 1, /* separateFiles */ 0, /* additionalParam */ 0, /* ldmFlag */ 0, /* ldmMinMatch */ @@ -239,7 +250,7 @@ static size_t local_defaultCompress( return out.pos; } -/* addiional argument is just the context */ +/* additional argument is just the context */ static size_t local_defaultDecompress( const void* srcBuffer, size_t srcSize, void* dstBuffer, size_t dstSize, @@ -269,14 +280,13 @@ static size_t local_defaultDecompress( /* initFn will be measured once, bench fn will be measured x times */ /* benchFn should return error value or out Size */ /* takes # of blocks and list of size & stuff for each. */ -BMK_customReturn_t BMK_benchCustom( - const char* functionName, size_t blockCount, - const void* const * const srcBuffers, const size_t* srcSizes, - void* const * const dstBuffers, const size_t* dstSizes, - size_t (*initFn)(void*), size_t (*benchFn)(const void*, size_t, void*, size_t, void*), - void* initPayload, void* benchPayload, - unsigned mode, unsigned iter, - int displayLevel) { +BMK_customReturn_t BMK_benchFunction( + size_t blockCount, + const void* const * const srcBlockBuffers, const size_t* srcBlockSizes, + void* const * const dstBlockBuffers, const size_t* dstBlockCapacities, + size_t (*initFn)(void*), void* initPayload, + size_t (*benchFn)(const void*, size_t, void*, size_t, void*), void* benchPayload, + unsigned mode, unsigned iter) { size_t srcSize = 0, dstSize = 0, ind = 0; unsigned toAdd = 1; @@ -287,24 +297,22 @@ BMK_customReturn_t BMK_benchCustom( { unsigned i; for(i = 0; i < blockCount; i++) { - memset(dstBuffers[i], 0xE5, dstSizes[i]); /* warm up and erase result buffer */ + memset(dstBlockBuffers[i], 0xE5, dstBlockCapacities[i]); /* warm up and erase result buffer */ } UTIL_sleepMilli(5); /* give processor time to other processes */ UTIL_waitForNextTick(); } - /* display last 17 char's of functionName*/ - if (strlen(functionName)>17) functionName += strlen(functionName)-17; if(!iter) { if(mode == BMK_iterMode) { - EXM_THROW(1, BMK_customReturn_t, "nbLoops must be nonzero \n"); + EXM_THROW_ND(1, BMK_customReturn_t, "nbLoops must be nonzero \n"); } } for(ind = 0; ind < blockCount; ind++) { - srcSize += srcSizes[ind]; + srcSize += srcBlockSizes[ind]; } switch(mode) { @@ -318,13 +326,13 @@ BMK_customReturn_t BMK_benchCustom( unsigned i, j; /* Overheat protection */ if (UTIL_clockSpanMicro(coolTime) > ACTIVEPERIOD_MICROSEC) { - DISPLAYLEVEL(2, "\rcooling down ... \r"); + DEBUGOUTPUT("\rcooling down ... \r"); UTIL_sleep(COOLPERIOD_SEC); coolTime = UTIL_getTime(); } for(i = 0; i < blockCount; i++) { - memset(dstBuffers[i], 0xD6, dstSizes[i]); /* warm up and erase result buffer */ + memset(dstBlockBuffers[i], 0xD6, dstBlockCapacities[i]); /* warm up and erase result buffer */ } clockStart = UTIL_getTime(); @@ -332,10 +340,10 @@ BMK_customReturn_t BMK_benchCustom( for(i = 0; i < nbLoops; i++) { for(j = 0; j < blockCount; j++) { - size_t res = (*benchFn)(srcBuffers[j], srcSizes[j], dstBuffers[j], dstSizes[j], benchPayload); + size_t res = (*benchFn)(srcBlockBuffers[j], srcBlockSizes[j], dstBlockBuffers[j], dstBlockCapacities[j], benchPayload); if(ZSTD_isError(res)) { - EXM_THROW(2, BMK_customReturn_t, "%s() failed on block %u of size %u : %s \n", - functionName, j, (U32)dstSizes[j], ZSTD_getErrorName(res)); + EXM_THROW_ND(2, BMK_customReturn_t, "Function benchmarking failed on block %u of size %u : %s \n", + j, (U32)dstBlockCapacities[j], ZSTD_getErrorName(res)); } else if (toAdd) { dstSize += res; } @@ -362,10 +370,10 @@ BMK_customReturn_t BMK_benchCustom( clockStart = UTIL_getTime(); for(i = 0; i < iter; i++) { for(j = 0; j < blockCount; j++) { - size_t res = (*benchFn)(srcBuffers[j], srcSizes[j], dstBuffers[j], dstSizes[j], benchPayload); + size_t res = (*benchFn)(srcBlockBuffers[j], srcBlockSizes[j], dstBlockBuffers[j], dstBlockCapacities[j], benchPayload); if(ZSTD_isError(res)) { - EXM_THROW(2, BMK_customReturn_t, "%s() failed on block %u of size %u : %s \n", - functionName, j, (U32)dstSizes[j], ZSTD_getErrorName(res)); + EXM_THROW_ND(2, BMK_customReturn_t, "Function benchmarking failed on block %u of size %u : %s \n", + j, (U32)dstBlockCapacities[j], ZSTD_getErrorName(res)); } else if(toAdd) { dstSize += res; } @@ -374,18 +382,18 @@ BMK_customReturn_t BMK_benchCustom( } totalTime = UTIL_clockSpanNano(clockStart); if(!totalTime) { - EXM_THROW(3, BMK_customReturn_t, "Cycle count (%u) too short to measure \n", iter); + EXM_THROW_ND(3, BMK_customReturn_t, "Cycle count (%u) too short to measure \n", iter); } else { fastest = totalTime / iter; } break; } default: - EXM_THROW(4, BMK_customReturn_t, "Unknown Mode \n"); + EXM_THROW_ND(4, BMK_customReturn_t, "Unknown Mode \n"); } retval.error = 0; - retval.result.time = fastest; - retval.result.size = dstSize; + retval.result.nanoSecPerRun = fastest; + retval.result.sumOfReturn = dstSize; return retval; } @@ -503,23 +511,23 @@ BMK_return_t BMK_benchMemAdvanced(const void* srcBuffer, size_t srcSize, cctxprep.adv = adv; /* Compression */ DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->\r", marks[markNb], displayName, (U32)srcSize); - compressionResults = BMK_benchCustom("ZSTD_compress_generic", nbBlocks, + compressionResults = BMK_benchFunction(nbBlocks, srcPtrs, srcSizes, cPtrs, cSizes, - &local_initCCtx, &local_defaultCompress, - (void*)&cctxprep, (void*)(ctx), - adv->loopMode, adv->nbSeconds, displayLevel); + &local_initCCtx, (void*)&cctxprep, + &local_defaultCompress, (void*)(ctx), + adv->loopMode, adv->nbSeconds); if(compressionResults.error) { results.error = compressionResults.error; return results; } - results.result.cSize = compressionResults.result.size; + results.result.cSize = compressionResults.result.sumOfReturn; ratio = (double)srcSize / (double)results.result.cSize; markNb = (markNb+1) % NB_MARKS; { int const ratioAccuracy = (ratio < 10.) ? 3 : 2; - double const compressionSpeed = ((double)srcSize / compressionResults.result.time) * 1000; + double const compressionSpeed = ((double)srcSize / compressionResults.result.nanoSecPerRun) * 1000; int const cSpeedAccuracy = (compressionSpeed < 10.) ? 2 : 1; results.result.cSpeed = compressionSpeed * 1000000; DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.*f),%6.*f MB/s\r", @@ -536,11 +544,11 @@ BMK_return_t BMK_benchMemAdvanced(const void* srcBuffer, size_t srcSize, dctxprep.dctx = dctx; dctxprep.dictBuffer = dictBuffer; dctxprep.dictBufferSize = dictBufferSize; - decompressionResults = BMK_benchCustom("ZSTD_decompress_generic", nbBlocks, + decompressionResults = BMK_benchFunction(nbBlocks, (const void * const *)cPtrs, cSizes, resPtrs, resSizes, - &local_initDCtx, &local_defaultDecompress, - (void*)&dctxprep, (void*)(dctx), - adv->loopMode, adv->nbSeconds, displayLevel); + &local_initDCtx, (void*)&dctxprep, + &local_defaultDecompress, (void*)(dctx), + adv->loopMode, adv->nbSeconds); if(decompressionResults.error) { results.error = decompressionResults.error; @@ -551,7 +559,7 @@ BMK_return_t BMK_benchMemAdvanced(const void* srcBuffer, size_t srcSize, { int const ratioAccuracy = (ratio < 10.) ? 3 : 2; double const compressionSpeed = results.result.cSpeed / 1000000; int const cSpeedAccuracy = (compressionSpeed < 10.) ? 2 : 1; - double const decompressionSpeed = ((double)srcSize / decompressionResults.result.time) * 1000; + double const decompressionSpeed = ((double)srcSize / decompressionResults.result.nanoSecPerRun) * 1000; results.result.dSpeed = decompressionSpeed * 1000000; DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.*f),%6.*f MB/s ,%6.1f MB/s \r", marks[markNb], displayName, (U32)srcSize, (U32)results.result.cSize, @@ -634,7 +642,7 @@ BMK_return_t BMK_benchMem(const void* srcBuffer, size_t srcSize, ZSTD_CCtx* ctx, ZSTD_DCtx* dctx, int displayLevel, const char* displayName) { - const BMK_advancedParams_t adv = BMK_defaultAdvancedParams(); + const BMK_advancedParams_t adv = BMK_initAdvancedParams(); return BMK_benchMemAdvanced(srcBuffer, srcSize, fileSizes, nbFiles, cLevel, comprParams, @@ -685,31 +693,20 @@ static size_t BMK_findMaxMem(U64 requiredMem) return (size_t)(requiredMem); } -ERROR_STRUCT(BMK_result_t*, BMK_returnPtr_t); - -/* returns average stats over all range [cLevel, cLevelLast] */ -static BMK_returnPtr_t BMK_benchCLevel(const void* srcBuffer, size_t benchedSize, +static BMK_return_t BMK_benchCLevel(const void* srcBuffer, size_t benchedSize, const size_t* fileSizes, unsigned nbFiles, - const int cLevel, const int cLevelLast, const ZSTD_compressionParameters* comprParams, + const int cLevel, const ZSTD_compressionParameters* comprParams, const void* dictBuffer, size_t dictBufferSize, int displayLevel, const char* displayName, BMK_advancedParams_t const * const adv) { - int l; - BMK_result_t* res = (BMK_result_t*)malloc(sizeof(BMK_result_t) * (cLevelLast - cLevel + 1)); - BMK_returnPtr_t ret; + BMK_return_t res; const char* pch = strrchr(displayName, '\\'); /* Windows */ - ret.error = 0; - ret.result = res; - if (!pch) pch = strrchr(displayName, '/'); /* Linux */ if (pch) displayName = pch+1; - if(res == NULL) { - EXM_THROW(12, BMK_returnPtr_t, "not enough memory\n"); - } if (adv->realTime) { DISPLAYLEVEL(2, "Note : switching to real-time priority \n"); SET_REALTIME_PRIORITY; @@ -718,23 +715,14 @@ static BMK_returnPtr_t BMK_benchCLevel(const void* srcBuffer, size_t benchedSize if (displayLevel == 1 && !adv->additionalParam) DISPLAY("bench %s %s: input %u bytes, %u seconds, %u KB blocks\n", ZSTD_VERSION_STRING, ZSTD_GIT_COMMIT_STRING, (U32)benchedSize, adv->nbSeconds, (U32)(adv->blockSize>>10)); - for (l=cLevel; l <= cLevelLast; l++) { - BMK_return_t rettmp; - if (l==0) continue; /* skip level 0 */ - rettmp = BMK_benchMemCtxless(srcBuffer, benchedSize, - fileSizes, nbFiles, - l, comprParams, - dictBuffer, dictBufferSize, - displayLevel, displayName, - adv); - if(rettmp.error) { - ret.error = rettmp.error; - return ret; - } - res[l-cLevel] = rettmp.result; - } + res = BMK_benchMemCtxless(srcBuffer, benchedSize, + fileSizes, nbFiles, + cLevel, comprParams, + dictBuffer, dictBufferSize, + displayLevel, displayName, + adv); - return ret; + return res; } @@ -776,8 +764,8 @@ static int BMK_loadFiles(void* buffer, size_t bufferSize, return 0; } -static BMK_returnSet_t BMK_benchFileTable(const char* const * const fileNamesTable, unsigned const nbFiles, - const char* const dictFileName, int const cLevel, int const cLevelLast, +static BMK_return_t BMK_benchFileTable(const char* const * const fileNamesTable, unsigned const nbFiles, + const char* const dictFileName, int const cLevel, const ZSTD_compressionParameters* const compressionParams, int displayLevel, const BMK_advancedParams_t * const adv) { @@ -786,23 +774,20 @@ static BMK_returnSet_t BMK_benchFileTable(const char* const * const fileNamesTab void* dictBuffer = NULL; size_t dictBufferSize = 0; size_t* const fileSizes = (size_t*)calloc(nbFiles, sizeof(size_t)); - BMK_returnSet_t res; + BMK_return_t res; U64 const totalSizeToLoad = UTIL_getTotalFileSize(fileNamesTable, nbFiles); - res.result.cLevel = cLevel; - res.result.cLevelLast = cLevelLast; - res.result.results = NULL; - if (!fileSizes) EXM_THROW(12, BMK_returnSet_t, "not enough memory for fileSizes"); + if (!fileSizes) EXM_THROW(12, BMK_return_t, "not enough memory for fileSizes"); /* Load dictionary */ if (dictFileName != NULL) { U64 const dictFileSize = UTIL_getFileSize(dictFileName); if (dictFileSize > 64 MB) - EXM_THROW(10, BMK_returnSet_t, "dictionary file %s too large", dictFileName); + EXM_THROW(10, BMK_return_t, "dictionary file %s too large", dictFileName); dictBufferSize = (size_t)dictFileSize; dictBuffer = malloc(dictBufferSize); if (dictBuffer==NULL) - EXM_THROW(11, BMK_returnSet_t, "not enough memory for dictionary (%u bytes)", + EXM_THROW(11, BMK_return_t, "not enough memory for dictionary (%u bytes)", (U32)dictBufferSize); { int errorCode = BMK_loadFiles(dictBuffer, dictBufferSize, fileSizes, &dictFileName, 1, displayLevel); @@ -819,7 +804,7 @@ static BMK_returnSet_t BMK_benchFileTable(const char* const * const fileNamesTab if (benchedSize < totalSizeToLoad) DISPLAY("Not enough memory; testing %u MB only...\n", (U32)(benchedSize >> 20)); srcBuffer = malloc(benchedSize); - if (!srcBuffer) EXM_THROW(12, BMK_returnSet_t, "not enough memory"); + if (!srcBuffer) EXM_THROW(12, BMK_return_t, "not enough memory"); /* Load input buffer */ { @@ -830,17 +815,18 @@ static BMK_returnSet_t BMK_benchFileTable(const char* const * const fileNamesTab } } /* Bench */ + /* if (adv->separateFiles) { const BYTE* srcPtr = (const BYTE*)srcBuffer; U32 fileNb; res.result.results = (BMK_result_t**)malloc(sizeof(BMK_result_t*) * nbFiles); res.result.nbFiles = nbFiles; - if(res.result.results == NULL) EXM_THROW(12, BMK_returnSet_t, "not enough memory"); + if(res.result.results == NULL) EXM_THROW(12, BMK_return_t, "not enough memory"); for (fileNb=0; fileNb 1) ? mfName : fileNamesTable[0]; - res.result.results = (BMK_result_t**)malloc(sizeof(BMK_result_t*)); - errorOrPtr = BMK_benchCLevel(srcBuffer, benchedSize, + res = BMK_benchCLevel(srcBuffer, benchedSize, fileSizes, nbFiles, - cLevel, cLevelLast, compressionParams, + cLevel, compressionParams, dictBuffer, dictBufferSize, displayLevel, displayName, adv); - if(res.result.results == NULL) EXM_THROW(12, BMK_returnSet_t, "not enough memory"); - if(errorOrPtr.error) { - res.error = errorOrPtr.error; - return res; - } - res.result.results[0] = errorOrPtr.result; } } /* clean up */ free(srcBuffer); free(dictBuffer); free(fileSizes); - res.error = 0; return res; } -static BMK_returnSet_t BMK_syntheticTest(int cLevel, int cLevelLast, double compressibility, +static BMK_return_t BMK_syntheticTest(int cLevel, double compressibility, const ZSTD_compressionParameters* compressionParams, int displayLevel, const BMK_advancedParams_t * const adv) { char name[20] = {0}; size_t benchedSize = 10000000; void* const srcBuffer = malloc(benchedSize); - BMK_returnSet_t res; - BMK_returnPtr_t errPtr; - res.result.results = (BMK_result_t**)calloc(1,sizeof(BMK_result_t*)); - res.result.nbFiles = 1; - res.result.cLevel = cLevel; - res.result.cLevelLast = cLevelLast; + BMK_return_t res; /* Memory allocation */ - if (!srcBuffer || !res.result.results) EXM_THROW(21, BMK_returnSet_t, "not enough memory"); + if (!srcBuffer) EXM_THROW(21, BMK_return_t, "not enough memory"); /* Fill input buffer */ RDG_genBuffer(srcBuffer, benchedSize, compressibility, 0.0, 0); /* Bench */ snprintf (name, sizeof(name), "Synthetic %2u%%", (unsigned)(compressibility*100)); - errPtr = BMK_benchCLevel(srcBuffer, benchedSize, + res = BMK_benchCLevel(srcBuffer, benchedSize, &benchedSize, 1, - cLevel, cLevelLast, compressionParams, + cLevel, compressionParams, NULL, 0, displayLevel, name, adv); - if(errPtr.error) { - res.error = errPtr.error; - return res; - } - res.result.results[0] = errPtr.result; /* clean up */ free((void*)srcBuffer); - res.error = 0; + return res; } -BMK_returnSet_t BMK_benchFilesAdvanced(const char** fileNamesTable, unsigned nbFiles, +BMK_return_t BMK_benchFilesAdvanced(const char** fileNamesTable, unsigned nbFiles, const char* dictFileName, - int cLevel, int cLevelLast, - const ZSTD_compressionParameters* compressionParams, + int cLevel, const ZSTD_compressionParameters* compressionParams, int displayLevel, const BMK_advancedParams_t * const adv) { double const compressibility = (double)g_compressibilityDefault / 100; if (cLevel > ZSTD_maxCLevel()) cLevel = ZSTD_maxCLevel(); - if (cLevelLast > ZSTD_maxCLevel()) cLevelLast = ZSTD_maxCLevel(); + /* if (cLevelLast > ZSTD_maxCLevel()) cLevelLast = ZSTD_maxCLevel(); if (cLevelLast < cLevel) cLevelLast = cLevel; - if (cLevelLast > cLevel) - DISPLAYLEVEL(2, "Benchmarking levels from %d to %d\n", cLevel, cLevelLast); + if (cLevelLast > cLevel) + DISPLAYLEVEL(2, "Benchmarking levels from %d to %d\n", cLevel, cLevelLast); */ if (nbFiles == 0) { - return BMK_syntheticTest(cLevel, cLevelLast, compressibility, compressionParams, displayLevel, adv); + return BMK_syntheticTest(cLevel, compressibility, compressionParams, displayLevel, adv); } else { - return BMK_benchFileTable(fileNamesTable, nbFiles, dictFileName, cLevel, cLevelLast, compressionParams, displayLevel, adv); + return BMK_benchFileTable(fileNamesTable, nbFiles, dictFileName, cLevel, compressionParams, displayLevel, adv); } } -int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles, +BMK_return_t BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles, const char* dictFileName, - int cLevel, int cLevelLast, - const ZSTD_compressionParameters* compressionParams, + int cLevel, const ZSTD_compressionParameters* compressionParams, int displayLevel) { - const BMK_advancedParams_t adv = BMK_defaultAdvancedParams(); - return BMK_benchFilesAdvanced(fileNamesTable, nbFiles, dictFileName, cLevel, cLevelLast, compressionParams, displayLevel, &adv).error; -} - -/* errorable or just return? */ -BMK_result_t BMK_getResult(BMK_resultSet_t resultSet, unsigned fileIdx, int cLevel) { - assert(resultSet.nbFiles > fileIdx); - assert(resultSet.cLevel <= cLevel && cLevel <= resultSet.cLevelLast); - return resultSet.results[fileIdx][cLevel - resultSet.cLevel]; -} - -void BMK_freeResultSet(BMK_resultSet_t src) { - unsigned i; - if(src.results == NULL) { return; } - for(i = 0; i < src.nbFiles; i++) { - free(src.results[i]); - } - free(src.results); + const BMK_advancedParams_t adv = BMK_initAdvancedParams(); + return BMK_benchFilesAdvanced(fileNamesTable, nbFiles, dictFileName, cLevel, compressionParams, displayLevel, &adv); } diff --git a/programs/bench.h b/programs/bench.h index 67430f33d..352166881 100644 --- a/programs/bench.h +++ b/programs/bench.h @@ -19,17 +19,12 @@ extern "C" { #define ZSTD_STATIC_LINKING_ONLY /* ZSTD_compressionParameters */ #include "zstd.h" /* ZSTD_compressionParameters */ -typedef enum { - BMK_timeMode = 0, - BMK_iterMode = 1 -} BMK_loopMode_t; - -typedef enum { - BMK_both = 0, - BMK_decodeOnly = 1, - BMK_compressOnly = 2 -} BMK_mode_t; - +/* Creates a struct of type typeName with an int type .error field + * and a .result field of some baseType. Functions with return + * typeName pass a successful result with .error = 0 and .result + * with the intended result, while returning an error will result + * in .error != 0. + */ #define ERROR_STRUCT(baseType, typeName) typedef struct { \ int error; \ baseType result; \ @@ -42,49 +37,50 @@ typedef struct { } BMK_result_t; typedef struct { - int cLevel; - int cLevelLast; - unsigned nbFiles; - BMK_result_t** results; -} BMK_resultSet_t; - -typedef struct { - size_t size; - U64 time; + size_t sumOfReturn; /* sum of return values */ + U64 nanoSecPerRun; /* time per iteration */ } BMK_customResult_t; ERROR_STRUCT(BMK_result_t, BMK_return_t); -ERROR_STRUCT(BMK_resultSet_t, BMK_returnSet_t); ERROR_STRUCT(BMK_customResult_t, BMK_customReturn_t); -/* want all 0 to be default, but wb ldmBucketSizeLog/ldmHashEveryLog */ +typedef enum { + BMK_timeMode = 0, + BMK_iterMode = 1 +} BMK_loopMode_t; + +typedef enum { + BMK_both = 0, + BMK_decodeOnly = 1, + BMK_compressOnly = 2 +} BMK_mode_t; + typedef struct { BMK_mode_t mode; /* 0: all, 1: compress only 2: decode only */ BMK_loopMode_t loopMode; /* if loopmode, then nbSeconds = nbLoops */ - unsigned nbSeconds; /* default timing is in nbSeconds. If nbCycles != 0 then use that */ + unsigned nbSeconds; /* default timing is in nbSeconds */ size_t blockSize; /* Maximum allowable size of a block*/ unsigned nbWorkers; /* multithreading */ - unsigned realTime; - unsigned separateFiles; - int additionalParam; - unsigned ldmFlag; - unsigned ldmMinMatch; - unsigned ldmHashLog; + unsigned realTime; /* real time priority */ + int additionalParam; /* used by python speed benchmark */ + unsigned ldmFlag; /* enables long distance matching */ + unsigned ldmMinMatch; /* below: parameters for long distance matching, see zstd.1.md for meaning */ + unsigned ldmHashLog; unsigned ldmBucketSizeLog; unsigned ldmHashEveryLog; } BMK_advancedParams_t; /* returns default parameters used by nonAdvanced functions */ -BMK_advancedParams_t BMK_defaultAdvancedParams(void); +BMK_advancedParams_t BMK_initAdvancedParams(void); -/* functionName - name of function - * blockCount - number of blocks (size of srcBuffers, srcSizes, dstBuffers, dstSizes) - * initFn - (*initFn)(initPayload) is run once per benchmark - * benchFn - (*benchFn)(srcBuffers[i], srcSizes[i], dstBuffers[i], dstSizes[i], benchPayload) - * is run a variable number of times, specified by mode and iter args - * mode - if 0, iter will be interpreted as the minimum number of seconds to run - * iter - see mode +/* called in cli */ +/* fileNamesTable - name of files to benchmark + * nbFiles - number of files (size of fileNamesTable) + * dictFileName - name of dictionary file to load + * cLevel - lowest compression level to benchmark + * cLevellast - highest compression level to benchmark (everything in the range [cLevel, cLevellast]) will be benchmarked + * compressionParams - basic compression Parameters * displayLevel - what gets printed * 0 : no display; * 1 : errors; @@ -92,16 +88,20 @@ BMK_advancedParams_t BMK_defaultAdvancedParams(void); * 3 : + progression; * 4 : + information * return - * .error will give a nonzero value if any error has occured - * .result will contain the speed (B/s) and time per loop (ns) + * .error will give a nonzero error value if an error has occured + * .result - if .error = 0, .result will return the time taken to compression speed + * (.cSpeed), decompression speed (.dSpeed), and copmressed size (.cSize) of the original + * file */ -BMK_customReturn_t BMK_benchCustom(const char* functionName, size_t blockCount, - const void* const * const srcBuffers, const size_t* srcSizes, - void* const * const dstBuffers, const size_t* dstSizes, - size_t (*initFn)(void*), size_t (*benchFn)(const void*, size_t, void*, size_t, void*), - void* initPayload, void* benchPayload, - unsigned mode, unsigned iter, - int displayLevel); +BMK_return_t BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles, const char* dictFileName, + int cLevel, const ZSTD_compressionParameters* compressionParams, + int displayLevel); + +/* See benchFiles for normal parameter uses and return, see advancedParams_t for adv */ +BMK_return_t BMK_benchFilesAdvanced(const char** fileNamesTable, unsigned nbFiles, + const char* dictFileName, + int cLevel, const ZSTD_compressionParameters* compressionParams, + int displayLevel, const BMK_advancedParams_t* const adv); /* basic benchmarking function, called in paramgrill ctx, dctx must be provided */ /* srcBuffer - data source, expected to be valid compressed data if in Decode Only Mode @@ -112,8 +112,12 @@ BMK_customReturn_t BMK_benchCustom(const char* functionName, size_t blockCount, * dictBufferSize - size of dictBuffer, 0 otherwise * ctx - Compression Context * dctx - Decompression Context - * diplayLevel - see BMK_benchCustom - * displayName - name used in display + * diplayLevel - see BMK_benchFiles + * displayName - name used by display + * return + * .error will give a nonzero value if an error has occured + * .result - if .error = 0, will give the same results as benchFiles + * but for the data stored in srcBuffer */ BMK_return_t BMK_benchMem(const void* srcBuffer, size_t srcSize, const size_t* fileSizes, unsigned nbFiles, @@ -122,6 +126,7 @@ BMK_return_t BMK_benchMem(const void* srcBuffer, size_t srcSize, ZSTD_CCtx* ctx, ZSTD_DCtx* dctx, int displayLevel, const char* displayName); +/* See benchMem for normal parameter uses and return, see advancedParams_t for adv */ BMK_return_t BMK_benchMemAdvanced(const void* srcBuffer, size_t srcSize, const size_t* fileSizes, unsigned nbFiles, const int cLevel, const ZSTD_compressionParameters* comprParams, @@ -130,29 +135,34 @@ BMK_return_t BMK_benchMemAdvanced(const void* srcBuffer, size_t srcSize, int displayLevel, const char* displayName, const BMK_advancedParams_t* adv); -/* called in cli */ -/* fileNamesTable - name of files to benchmark - * nbFiles - number of files (size of fileNamesTable) - * dictFileName - name of dictionary file to load - * cLevel - lowest compression level to benchmark - * cLevellast - highest compression level to benchmark (everything in the range [cLevel, cLevellast]) will be benchmarked - * compressionParams - basic compression Parameters - * displayLevel - see BMK_benchCustom +/* This function benchmarks the running time two functions (function specifics described */ + +/* blockCount - number of blocks (size of srcBuffers, srcSizes, dstBuffers, dstCapacities) + * srcBuffers - an array of buffers to be operated on by benchFn + * srcSizes - an array of the sizes of above buffers + * dstBuffers - an array of buffers to be written into by benchFn + * dstCapacities - an array of the capacities of above buffers. + * initFn - (*initFn)(initPayload) is run once per benchmark + * benchFn - (*benchFn)(srcBuffers[i], srcSizes[i], dstBuffers[i], dstCapacities[i], benchPayload) + * is run a variable number of times, specified by mode and iter args + * mode - if 0, iter will be interpreted as the minimum number of seconds to run + * iter - see mode + * return + * .error will give a nonzero value if ZSTD_isError() is nonzero for any of the return + * of the calls to initFn and benchFn, or if benchFunction errors internally + * .result - if .error = 0, then .result will contain the sum of all return values of + * benchFn on the first iteration through all of the blocks (.sumOfReturn) and also + * the time per run of benchFn (.nanoSecPerRun). For the former, this + * is generally intended to be used on functions which return the # of bytes written + * into dstBuffer, hence this value will be the total amount of bytes written to + * dstBuffer. */ -int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles, const char* dictFileName, - int cLevel, int cLevelLast, const ZSTD_compressionParameters* compressionParams, - int displayLevel); - -BMK_returnSet_t BMK_benchFilesAdvanced(const char** fileNamesTable, unsigned nbFiles, - const char* dictFileName, - int cLevel, int cLevelLast, - const ZSTD_compressionParameters* compressionParams, - int displayLevel, const BMK_advancedParams_t* adv); - -/* get data from resultSet */ -/* when aggregated (separateFiles = 0), just be getResult(r,0,cl) */ -BMK_result_t BMK_getResult(BMK_resultSet_t results, unsigned fileIdx, int cLevel); -void BMK_freeResultSet(BMK_resultSet_t src); +BMK_customReturn_t BMK_benchFunction(size_t blockCount, + const void* const * const srcBuffers, const size_t* srcSizes, + void* const * const dstBuffers, const size_t* dstCapacities, + size_t (*initFn)(void*), void* initPayload, + size_t (*benchFn)(const void*, size_t, void*, size_t, void*), void* benchPayload, + unsigned mode, unsigned iter); #endif /* BENCH_H_121279284357 */ diff --git a/programs/zstdcli.c b/programs/zstdcli.c index 8b31a98b1..61a43dc30 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -398,7 +398,7 @@ int main(int argCount, const char* argv[]) setRealTimePrio = 0, singleThread = 0, ultra=0; - BMK_advancedParams_t adv = BMK_defaultAdvancedParams(); + BMK_advancedParams_t adv = BMK_initAdvancedParams(); unsigned bench_nbSeconds = 3; /* would be better if this value was synchronized from bench */ size_t blockSize = 0; zstd_operation_mode operation = zom_compress; @@ -802,7 +802,6 @@ int main(int argCount, const char* argv[]) /* Check if benchmark is selected */ if (operation==zom_bench) { #ifndef ZSTD_NOBENCH - adv.separateFiles = separateFiles; adv.blockSize = blockSize; adv.nbWorkers = nbWorkers; adv.realTime = setRealTimePrio; @@ -816,7 +815,18 @@ int main(int argCount, const char* argv[]) if (g_ldmHashEveryLog != LDM_PARAM_DEFAULT) { adv.ldmHashEveryLog = g_ldmHashEveryLog; } - BMK_freeResultSet(BMK_benchFilesAdvanced(filenameTable, filenameIdx, dictFileName, cLevel, cLevelLast, &compressionParams, g_displayLevel, &adv).result); + + for(; cLevel <= cLevelLast; cLevel++) { + if(separateFiles) { + unsigned i; + for(i = 0; i < filenameIdx; i++) { + BMK_benchFilesAdvanced(&filenameTable[i], 1, dictFileName, cLevel, &compressionParams, g_displayLevel, &adv); + } + } else { + BMK_benchFilesAdvanced(filenameTable, filenameIdx, dictFileName, cLevel, &compressionParams, g_displayLevel, &adv); + } + } + #else (void)bench_nbSeconds; (void)blockSize; (void)setRealTimePrio; (void)separateFiles; #endif diff --git a/tests/fullbench.c b/tests/fullbench.c index 2dee3db94..b83eb1c77 100644 --- a/tests/fullbench.c +++ b/tests/fullbench.c @@ -431,14 +431,16 @@ static size_t benchMem(const void* src, size_t srcSize, U32 benchNb) /* benchmark loop */ { - BMK_customReturn_t r = BMK_benchCustom(benchName, 1, &src, &srcSize, (void * const * const)&dstBuff, &dstBuffSize, &local_nothing, benchFunction, - NULL, buff2, BMK_timeMode, 1, 2); + BMK_customReturn_t r = BMK_benchFunction(1, &src, &srcSize, + (void * const * const)&dstBuff, &dstBuffSize, + &local_nothing, NULL, + benchFunction, buff2, BMK_timeMode, 1); if(r.error) { DISPLAY("ERROR %d ! ! \n", r.error); exit(1); } - DISPLAY("%2u#Speed: %f MB/s - Size: %f MB\n", benchNb, (double)srcSize / r.result.time * 1000, (double)r.result.size / 1000000); + DISPLAY("%2u#Speed: %f MB/s - Size: %f MB - %s\n", benchNb, (double)srcSize / r.result.nanoSecPerRun * 1000, (double)r.result.sumOfReturn / 1000000, benchName); } _cleanOut: From e482e328cddc9a654185d414bb1e1925ec47046d Mon Sep 17 00:00:00 2001 From: George Lu Date: Mon, 18 Jun 2018 12:08:51 -0700 Subject: [PATCH 005/372] Reorder Arguments make initFn nullable --- programs/bench.c | 21 ++++++++++++--------- programs/bench.h | 16 +++++++++------- tests/fullbench.c | 12 +++++------- 3 files changed, 26 insertions(+), 23 deletions(-) diff --git a/programs/bench.c b/programs/bench.c index 6d0e1c1ff..fae4ea0fa 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -281,11 +281,11 @@ static size_t local_defaultDecompress( /* benchFn should return error value or out Size */ /* takes # of blocks and list of size & stuff for each. */ BMK_customReturn_t BMK_benchFunction( + size_t (*benchFn)(const void*, size_t, void*, size_t, void*), void* benchPayload, + size_t (*initFn)(void*), void* initPayload, size_t blockCount, const void* const * const srcBlockBuffers, const size_t* srcBlockSizes, void* const * const dstBlockBuffers, const size_t* dstBlockCapacities, - size_t (*initFn)(void*), void* initPayload, - size_t (*benchFn)(const void*, size_t, void*, size_t, void*), void* benchPayload, unsigned mode, unsigned iter) { size_t srcSize = 0, dstSize = 0, ind = 0; unsigned toAdd = 1; @@ -336,7 +336,7 @@ BMK_customReturn_t BMK_benchFunction( } clockStart = UTIL_getTime(); - (*initFn)(initPayload); + if(initFn != NULL) { (*initFn)(initPayload); } for(i = 0; i < nbLoops; i++) { for(j = 0; j < blockCount; j++) { @@ -368,6 +368,7 @@ BMK_customReturn_t BMK_benchFunction( { unsigned i, j; clockStart = UTIL_getTime(); + if(initFn != NULL) { (*initFn)(initPayload); } for(i = 0; i < iter; i++) { for(j = 0; j < blockCount; j++) { size_t res = (*benchFn)(srcBlockBuffers[j], srcBlockSizes[j], dstBlockBuffers[j], dstBlockCapacities[j], benchPayload); @@ -511,10 +512,11 @@ BMK_return_t BMK_benchMemAdvanced(const void* srcBuffer, size_t srcSize, cctxprep.adv = adv; /* Compression */ DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->\r", marks[markNb], displayName, (U32)srcSize); - compressionResults = BMK_benchFunction(nbBlocks, - srcPtrs, srcSizes, cPtrs, cSizes, - &local_initCCtx, (void*)&cctxprep, + compressionResults = BMK_benchFunction( &local_defaultCompress, (void*)(ctx), + &local_initCCtx, (void*)&cctxprep, + nbBlocks, + srcPtrs, srcSizes, cPtrs, cSizes, adv->loopMode, adv->nbSeconds); if(compressionResults.error) { @@ -544,10 +546,11 @@ BMK_return_t BMK_benchMemAdvanced(const void* srcBuffer, size_t srcSize, dctxprep.dctx = dctx; dctxprep.dictBuffer = dictBuffer; dctxprep.dictBufferSize = dictBufferSize; - decompressionResults = BMK_benchFunction(nbBlocks, - (const void * const *)cPtrs, cSizes, resPtrs, resSizes, - &local_initDCtx, (void*)&dctxprep, + decompressionResults = BMK_benchFunction( &local_defaultDecompress, (void*)(dctx), + &local_initDCtx, (void*)&dctxprep, + nbBlocks, + (const void * const *)cPtrs, cSizes, resPtrs, resSizes, adv->loopMode, adv->nbSeconds); if(decompressionResults.error) { diff --git a/programs/bench.h b/programs/bench.h index 352166881..2030f0a2f 100644 --- a/programs/bench.h +++ b/programs/bench.h @@ -137,14 +137,15 @@ BMK_return_t BMK_benchMemAdvanced(const void* srcBuffer, size_t srcSize, /* This function benchmarks the running time two functions (function specifics described */ -/* blockCount - number of blocks (size of srcBuffers, srcSizes, dstBuffers, dstCapacities) +/* benchFn - (*benchFn)(srcBuffers[i], srcSizes[i], dstBuffers[i], dstCapacities[i], benchPayload) + * is run a variable number of times, specified by mode and iter args + * initFn - (*initFn)(initPayload) is run once per benchmark at the beginning. This argument can + * be NULL, in which case nothing is run. + * blockCount - number of blocks (size of srcBuffers, srcSizes, dstBuffers, dstCapacities) * srcBuffers - an array of buffers to be operated on by benchFn * srcSizes - an array of the sizes of above buffers * dstBuffers - an array of buffers to be written into by benchFn * dstCapacities - an array of the capacities of above buffers. - * initFn - (*initFn)(initPayload) is run once per benchmark - * benchFn - (*benchFn)(srcBuffers[i], srcSizes[i], dstBuffers[i], dstCapacities[i], benchPayload) - * is run a variable number of times, specified by mode and iter args * mode - if 0, iter will be interpreted as the minimum number of seconds to run * iter - see mode * return @@ -157,11 +158,12 @@ BMK_return_t BMK_benchMemAdvanced(const void* srcBuffer, size_t srcSize, * into dstBuffer, hence this value will be the total amount of bytes written to * dstBuffer. */ -BMK_customReturn_t BMK_benchFunction(size_t blockCount, +BMK_customReturn_t BMK_benchFunction( + size_t (*benchFn)(const void*, size_t, void*, size_t, void*), void* benchPayload, + size_t (*initFn)(void*), void* initPayload, + size_t blockCount, const void* const * const srcBuffers, const size_t* srcSizes, void* const * const dstBuffers, const size_t* dstCapacities, - size_t (*initFn)(void*), void* initPayload, - size_t (*benchFn)(const void*, size_t, void*, size_t, void*), void* benchPayload, unsigned mode, unsigned iter); #endif /* BENCH_H_121279284357 */ diff --git a/tests/fullbench.c b/tests/fullbench.c index b83eb1c77..91a1370df 100644 --- a/tests/fullbench.c +++ b/tests/fullbench.c @@ -94,10 +94,6 @@ static size_t BMK_findMaxMem(U64 requiredMem) /*_******************************************************* * Benchmark wrappers *********************************************************/ -size_t local_nothing(void* x) { - (void)x; - return 0; -} size_t local_ZSTD_compress(const void* src, size_t srcSize, void* dst, size_t dstSize, void* buff2) { @@ -431,10 +427,12 @@ static size_t benchMem(const void* src, size_t srcSize, U32 benchNb) /* benchmark loop */ { - BMK_customReturn_t r = BMK_benchFunction(1, &src, &srcSize, + BMK_customReturn_t r = BMK_benchFunction( + benchFunction, buff2, + NULL, NULL, + 1, &src, &srcSize, (void * const * const)&dstBuff, &dstBuffSize, - &local_nothing, NULL, - benchFunction, buff2, BMK_timeMode, 1); + BMK_timeMode, 1); if(r.error) { DISPLAY("ERROR %d ! ! \n", r.error); exit(1); From a3c8b599901cd3e4a517c066f27c9196ea8c6176 Mon Sep 17 00:00:00 2001 From: George Lu Date: Mon, 18 Jun 2018 15:06:31 -0700 Subject: [PATCH 006/372] Fix cli no print Change looping behavior to match old --- programs/bench.c | 33 +++------------------------------ programs/bench.h | 16 ++++++++++------ programs/zstdcli.c | 26 ++++++++++++++++++-------- 3 files changed, 31 insertions(+), 44 deletions(-) diff --git a/programs/bench.c b/programs/bench.c index fae4ea0fa..a9a8086b2 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -818,30 +818,6 @@ static BMK_return_t BMK_benchFileTable(const char* const * const fileNamesTable, } } /* Bench */ - /* - if (adv->separateFiles) { - const BYTE* srcPtr = (const BYTE*)srcBuffer; - U32 fileNb; - res.result.results = (BMK_result_t**)malloc(sizeof(BMK_result_t*) * nbFiles); - res.result.nbFiles = nbFiles; - if(res.result.results == NULL) EXM_THROW(12, BMK_return_t, "not enough memory"); - for (fileNb=0; fileNb ZSTD_maxCLevel()) cLevel = ZSTD_maxCLevel(); - /* if (cLevelLast > ZSTD_maxCLevel()) cLevelLast = ZSTD_maxCLevel(); - if (cLevelLast < cLevel) cLevelLast = cLevel; - if (cLevelLast > cLevel) - DISPLAYLEVEL(2, "Benchmarking levels from %d to %d\n", cLevel, cLevelLast); */ - + if (cLevel > ZSTD_maxCLevel()) { + EXM_THROW(15, BMK_return_t, "Invalid Compression Level"); + } if (nbFiles == 0) { return BMK_syntheticTest(cLevel, compressibility, compressionParams, displayLevel, adv); } diff --git a/programs/bench.h b/programs/bench.h index 2030f0a2f..d553e7b4f 100644 --- a/programs/bench.h +++ b/programs/bench.h @@ -75,11 +75,12 @@ typedef struct { BMK_advancedParams_t BMK_initAdvancedParams(void); /* called in cli */ +/* Loads files in fileNamesTable into memory, as well as a dictionary + * from dictFileName, and then uses benchMem */ /* fileNamesTable - name of files to benchmark * nbFiles - number of files (size of fileNamesTable) * dictFileName - name of dictionary file to load - * cLevel - lowest compression level to benchmark - * cLevellast - highest compression level to benchmark (everything in the range [cLevel, cLevellast]) will be benchmarked + * cLevel - compression level to benchmark, errors if invalid * compressionParams - basic compression Parameters * displayLevel - what gets printed * 0 : no display; @@ -103,15 +104,18 @@ BMK_return_t BMK_benchFilesAdvanced(const char** fileNamesTable, unsigned nbFile int cLevel, const ZSTD_compressionParameters* compressionParams, int displayLevel, const BMK_advancedParams_t* const adv); -/* basic benchmarking function, called in paramgrill ctx, dctx must be provided */ +/* basic benchmarking function, called in paramgrill + * applies ZSTD_compress_generic() and ZSTD_decompress_generic() on data in srcBuffer + * with specific compression parameters specified by other arguments using benchFunction + * (cLevel, comprParams + adv in advanced Mode) */ /* srcBuffer - data source, expected to be valid compressed data if in Decode Only Mode * srcSize - size of data in srcBuffer * cLevel - compression level * comprParams - basic compression parameters * dictBuffer - a dictionary if used, null otherwise * dictBufferSize - size of dictBuffer, 0 otherwise - * ctx - Compression Context - * dctx - Decompression Context + * ctx - Compression Context (must be provided) + * dctx - Decompression Context (must be provided) * diplayLevel - see BMK_benchFiles * displayName - name used by display * return @@ -135,7 +139,7 @@ BMK_return_t BMK_benchMemAdvanced(const void* srcBuffer, size_t srcSize, int displayLevel, const char* displayName, const BMK_advancedParams_t* adv); -/* This function benchmarks the running time two functions (function specifics described */ +/* This function times the execution of 2 argument functions, benchFn and initFn */ /* benchFn - (*benchFn)(srcBuffers[i], srcSizes[i], dstBuffers[i], dstCapacities[i], benchPayload) * is run a variable number of times, specified by mode and iter args diff --git a/programs/zstdcli.c b/programs/zstdcli.c index 61a43dc30..63be9ef61 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -32,7 +32,7 @@ #include /* errno */ #include "fileio.h" /* stdinmark, stdoutmark, ZSTD_EXTENSION */ #ifndef ZSTD_NOBENCH -# include "bench.h" /* BMK_benchFiles, BMK_SetNbSeconds */ +# include "bench.h" /* BMK_benchFiles */ #endif #ifndef ZSTD_NODICT # include "dibio.h" /* ZDICT_cover_params_t, DiB_trainFromFiles() */ @@ -816,15 +816,25 @@ int main(int argCount, const char* argv[]) adv.ldmHashEveryLog = g_ldmHashEveryLog; } - for(; cLevel <= cLevelLast; cLevel++) { - if(separateFiles) { - unsigned i; - for(i = 0; i < filenameIdx; i++) { - BMK_benchFilesAdvanced(&filenameTable[i], 1, dictFileName, cLevel, &compressionParams, g_displayLevel, &adv); + if (cLevel > ZSTD_maxCLevel()) cLevel = ZSTD_maxCLevel(); + if (cLevelLast > ZSTD_maxCLevel()) cLevelLast = ZSTD_maxCLevel(); + if (cLevelLast < cLevel) cLevelLast = cLevel; + if (cLevelLast > cLevel) + DISPLAYLEVEL(2, "Benchmarking levels from %d to %d\n", cLevel, cLevelLast); + + if(separateFiles) { + unsigned i; + for(i = 0; i < filenameIdx; i++) { + DISPLAYLEVEL(2, "Benchmarking %s \n", filenameTable[i]); + int c; + for(c = cLevel; c <= cLevelLast; c++) { + BMK_benchFilesAdvanced(&filenameTable[i], 1, dictFileName, c, &compressionParams, g_displayLevel, &adv); } - } else { - BMK_benchFilesAdvanced(filenameTable, filenameIdx, dictFileName, cLevel, &compressionParams, g_displayLevel, &adv); } + } else { + for(; cLevel <= cLevelLast; cLevel++) { + BMK_benchFilesAdvanced(filenameTable, filenameIdx, dictFileName, cLevel, &compressionParams, g_displayLevel, &adv); + } } #else From a8eea99ebe00eb5bfcf7d3689faf55128931ad00 Mon Sep 17 00:00:00 2001 From: George Lu Date: Tue, 19 Jun 2018 10:58:22 -0700 Subject: [PATCH 007/372] Incremental Display + Fn Separations Seperate syntheticTest and fileTableTest (now renamed as benchFiles) Add incremental display to benchMem Change to only iterMode for benchFunction Make Synthetic test's compressibility configurable from cli (using -P#) --- programs/bench.c | 450 ++++++++++++++++++++++++++------------------- programs/bench.h | 40 ++-- programs/zstdcli.c | 39 +++- tests/fullbench.c | 16 +- 4 files changed, 326 insertions(+), 219 deletions(-) diff --git a/programs/bench.c b/programs/bench.c index a9a8086b2..447f9feb9 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -64,9 +64,6 @@ static const size_t maxMemory = (sizeof(size_t)==4) ? (2 GB - 64 MB) : (size_t)(1ULL << ((sizeof(size_t)*8)-31)); -/* remove this in the future? */ -static U32 g_compressibilityDefault = 50; - /* ************************************* * console display ***************************************/ @@ -276,22 +273,23 @@ static size_t local_defaultDecompress( } -/* mode 0 : iter = # seconds, else iter = # cycles */ /* initFn will be measured once, bench fn will be measured x times */ /* benchFn should return error value or out Size */ /* takes # of blocks and list of size & stuff for each. */ +/* only does iterations*/ +/* note time/iter could be zero if interval too short */ BMK_customReturn_t BMK_benchFunction( size_t (*benchFn)(const void*, size_t, void*, size_t, void*), void* benchPayload, size_t (*initFn)(void*), void* initPayload, size_t blockCount, const void* const * const srcBlockBuffers, const size_t* srcBlockSizes, void* const * const dstBlockBuffers, const size_t* dstBlockCapacities, - unsigned mode, unsigned iter) { + unsigned iter) { size_t srcSize = 0, dstSize = 0, ind = 0; unsigned toAdd = 1; + U64 totalTime; BMK_customReturn_t retval; - U64 totalTime = 0, fastest = (U64)(-1LL); UTIL_time_t clockStart; { @@ -305,136 +303,60 @@ BMK_customReturn_t BMK_benchFunction( } if(!iter) { - if(mode == BMK_iterMode) { - EXM_THROW_ND(1, BMK_customReturn_t, "nbLoops must be nonzero \n"); - } - + EXM_THROW_ND(1, BMK_customReturn_t, "nbLoops must be nonzero \n"); } for(ind = 0; ind < blockCount; ind++) { srcSize += srcBlockSizes[ind]; } - switch(mode) { - case BMK_timeMode: - { - int completed = 0; - U64 const maxTime = (iter * TIMELOOP_NANOSEC) + 1; - unsigned nbLoops = 1; - UTIL_time_t coolTime = UTIL_getTime(); - while(!completed) { - unsigned i, j; - /* Overheat protection */ - if (UTIL_clockSpanMicro(coolTime) > ACTIVEPERIOD_MICROSEC) { - DEBUGOUTPUT("\rcooling down ... \r"); - UTIL_sleep(COOLPERIOD_SEC); - coolTime = UTIL_getTime(); + { + unsigned i, j; + clockStart = UTIL_getTime(); + if(initFn != NULL) { (*initFn)(initPayload); } + for(i = 0; i < iter; i++) { + for(j = 0; j < blockCount; j++) { + size_t res = (*benchFn)(srcBlockBuffers[j], srcBlockSizes[j], dstBlockBuffers[j], dstBlockCapacities[j], benchPayload); + if(ZSTD_isError(res)) { + EXM_THROW_ND(2, BMK_customReturn_t, "Function benchmarking failed on block %u of size %u : %s \n", + j, (U32)dstBlockCapacities[j], ZSTD_getErrorName(res)); + } else if(toAdd) { + dstSize += res; } - - for(i = 0; i < blockCount; i++) { - memset(dstBlockBuffers[i], 0xD6, dstBlockCapacities[i]); /* warm up and erase result buffer */ - } - - clockStart = UTIL_getTime(); - if(initFn != NULL) { (*initFn)(initPayload); } - - for(i = 0; i < nbLoops; i++) { - for(j = 0; j < blockCount; j++) { - size_t res = (*benchFn)(srcBlockBuffers[j], srcBlockSizes[j], dstBlockBuffers[j], dstBlockCapacities[j], benchPayload); - if(ZSTD_isError(res)) { - EXM_THROW_ND(2, BMK_customReturn_t, "Function benchmarking failed on block %u of size %u : %s \n", - j, (U32)dstBlockCapacities[j], ZSTD_getErrorName(res)); - } else if (toAdd) { - dstSize += res; - } - } - toAdd = 0; - } - { U64 const loopDuration = UTIL_clockSpanNano(clockStart); - if (loopDuration > 0) { - fastest = MIN(fastest, loopDuration / nbLoops); - nbLoops = (U32)(TIMELOOP_NANOSEC / fastest) + 1; - } else { - assert(nbLoops < 40000000); /* avoid overflow */ - nbLoops *= 100; - } - totalTime += loopDuration; - completed = (totalTime >= maxTime); - } } - break; + toAdd = 0; } - case BMK_iterMode: - { - unsigned i, j; - clockStart = UTIL_getTime(); - if(initFn != NULL) { (*initFn)(initPayload); } - for(i = 0; i < iter; i++) { - for(j = 0; j < blockCount; j++) { - size_t res = (*benchFn)(srcBlockBuffers[j], srcBlockSizes[j], dstBlockBuffers[j], dstBlockCapacities[j], benchPayload); - if(ZSTD_isError(res)) { - EXM_THROW_ND(2, BMK_customReturn_t, "Function benchmarking failed on block %u of size %u : %s \n", - j, (U32)dstBlockCapacities[j], ZSTD_getErrorName(res)); - } else if(toAdd) { - dstSize += res; - } - } - toAdd = 0; - } - totalTime = UTIL_clockSpanNano(clockStart); - if(!totalTime) { - EXM_THROW_ND(3, BMK_customReturn_t, "Cycle count (%u) too short to measure \n", iter); - } else { - fastest = totalTime / iter; - } - break; - } - default: - EXM_THROW_ND(4, BMK_customReturn_t, "Unknown Mode \n"); + totalTime = UTIL_clockSpanNano(clockStart); } + retval.error = 0; - retval.result.nanoSecPerRun = fastest; + retval.result.nanoSecPerRun = totalTime / iter; retval.result.sumOfReturn = dstSize; return retval; } -BMK_return_t BMK_benchMemAdvanced(const void* srcBuffer, size_t srcSize, - const size_t* fileSizes, unsigned nbFiles, - const int cLevel, const ZSTD_compressionParameters* comprParams, - const void* dictBuffer, size_t dictBufferSize, - ZSTD_CCtx* ctx, ZSTD_DCtx* dctx, - int displayLevel, const char* displayName, const BMK_advancedParams_t* adv) +/* benchMem with no allocation */ +static BMK_return_t BMK_benchMemAdvancedNoAlloc( + const void ** const srcPtrs, size_t* const srcSizes, + void** const cPtrs, size_t* const cSizes, + void** const resPtrs, size_t* const resSizes, + void* resultBuffer, void* compressedBuffer, + const size_t maxCompressedSize, + const void* srcBuffer, size_t srcSize, + const size_t* fileSizes, unsigned nbFiles, + const int cLevel, const ZSTD_compressionParameters* comprParams, + const void* dictBuffer, size_t dictBufferSize, + ZSTD_CCtx* ctx, ZSTD_DCtx* dctx, + int displayLevel, const char* displayName, const BMK_advancedParams_t* adv) { size_t const blockSize = ((adv->blockSize>=32 && (adv->mode != BMK_decodeOnly)) ? adv->blockSize : srcSize) + (!srcSize) /* avoid div by 0 */ ; - U32 const maxNbBlocks = (U32) ((srcSize + (blockSize-1)) / blockSize) + nbFiles; - - /* these are the blockTable parameters, just split up */ - const void ** const srcPtrs = (const void ** const)malloc(maxNbBlocks * sizeof(void*)); - size_t* const srcSizes = (size_t* const)malloc(maxNbBlocks * sizeof(size_t)); - - void ** const cPtrs = (void** const)malloc(maxNbBlocks * sizeof(void*)); - size_t* const cSizes = (size_t* const)malloc(maxNbBlocks * sizeof(size_t)); - - void ** const resPtrs = (void** const)malloc(maxNbBlocks * sizeof(void*)); - size_t* const resSizes = (size_t* const)malloc(maxNbBlocks * sizeof(size_t)); - - const size_t maxCompressedSize = ZSTD_compressBound(srcSize) + (maxNbBlocks * 1024); /* add some room for safety */ - void* compressedBuffer = malloc(maxCompressedSize); - void* resultBuffer = malloc(srcSize); - BMK_return_t results; - size_t const loadedCompressedSize = srcSize; size_t cSize = 0; double ratio = 0.; U32 nbBlocks; - /* checks */ - if (!compressedBuffer || !resultBuffer || - !srcPtrs || !srcSizes || !cPtrs || !cSizes || !resPtrs || !resSizes) - EXM_THROW(31, BMK_return_t, "allocation error : not enough memory"); - if(!ctx || !dctx) EXM_THROW(31, BMK_return_t, "error: passed in null context"); @@ -451,10 +373,15 @@ BMK_return_t BMK_benchMemAdvanced(const void* srcBuffer, size_t srcSize, srcPtr += fileSizes[fileNb]; } { size_t const decodedSize = (size_t)totalDSize64; - if (totalDSize64 > decodedSize) EXM_THROW(32, BMK_return_t, "original size is too large"); /* size_t overflow */ free(resultBuffer); resultBuffer = malloc(decodedSize); - if (!resultBuffer) EXM_THROW(33, BMK_return_t, "not enough memory"); + if (!resultBuffer) { + EXM_THROW(33, BMK_return_t, "not enough memory"); + } + if (totalDSize64 > decodedSize) { + free(resultBuffer); + EXM_THROW(32, BMK_return_t, "original size is too large"); /* size_t overflow */ + } cSize = srcSize; srcSize = decodedSize; ratio = (double)srcSize / (double)cSize; @@ -504,6 +431,9 @@ BMK_return_t BMK_benchMemAdvanced(const void* srcBuffer, size_t srcSize, if (adv->mode != BMK_decodeOnly) { BMK_initCCtxArgs cctxprep; BMK_customReturn_t compressionResults; + int completed = 0; + U64 totalLoops = 0, totalTime = 0, fastest = (U64)(-1LL); + UTIL_time_t coolTime = UTIL_getTime(); cctxprep.ctx = ctx; cctxprep.dictBuffer = dictBuffer; cctxprep.dictBufferSize = dictBufferSize; @@ -512,63 +442,161 @@ BMK_return_t BMK_benchMemAdvanced(const void* srcBuffer, size_t srcSize, cctxprep.adv = adv; /* Compression */ DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->\r", marks[markNb], displayName, (U32)srcSize); - compressionResults = BMK_benchFunction( - &local_defaultCompress, (void*)(ctx), - &local_initCCtx, (void*)&cctxprep, - nbBlocks, - srcPtrs, srcSizes, cPtrs, cSizes, - adv->loopMode, adv->nbSeconds); + if(adv->loopMode == BMK_timeMode) { + U64 maxTime = adv->nbSeconds * TIMELOOP_NANOSEC; + unsigned nbLoops = 1; + while(!completed) { + /* Overheat protection */ + if (UTIL_clockSpanMicro(coolTime) > ACTIVEPERIOD_MICROSEC) { + DEBUGOUTPUT("\rcooling down ... \r"); + UTIL_sleep(COOLPERIOD_SEC); + coolTime = UTIL_getTime(); + } - if(compressionResults.error) { - results.error = compressionResults.error; - return results; + compressionResults = BMK_benchFunction(&local_defaultCompress, (void*)ctx, &local_initCCtx, (void*)&cctxprep, + nbBlocks, srcPtrs, srcSizes, cPtrs, cSizes, nbLoops); + if(compressionResults.error) { + results.error = compressionResults.error; + return results; + } + + { U64 loopDuration = compressionResults.result.nanoSecPerRun * nbLoops; + totalLoops += nbLoops; + totalTime += loopDuration; + if (loopDuration > 0) { // nanoSec / run + fastest = MIN(fastest, compressionResults.result.nanoSecPerRun); + nbLoops = (U32)(TIMELOOP_NANOSEC / fastest) + 1; + } else { + assert(nbLoops < 40000000); /* avoid overflow */ + nbLoops *= 2; + } + completed = (totalTime >= maxTime); + { + int const ratioAccuracy = (ratio < 10.) ? 3 : 2; + double const compressionSpeed = (((double)srcSize * totalLoops) / totalTime) * 1000; + int const cSpeedAccuracy = (compressionSpeed < 10.) ? 2 : 1; + results.result.cSpeed = compressionSpeed * 1000000; + results.result.cSize = compressionResults.result.sumOfReturn; + ratio = (double)srcSize / results.result.cSize; + markNb = (markNb+1) % NB_MARKS; + DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.*f),%6.*f MB/s\r", + marks[markNb], displayName, (U32)srcSize, (U32)results.result.cSize, + ratioAccuracy, ratio, + cSpeedAccuracy, compressionSpeed); + } + } + } + } else { + compressionResults = BMK_benchFunction(&local_defaultCompress, (void*)ctx, &local_initCCtx, (void*)&cctxprep, + nbBlocks, srcPtrs, srcSizes, cPtrs, cSizes, adv->nbSeconds); + if(compressionResults.error) { + results.error = compressionResults.error; + return results; + } + if(compressionResults.result.nanoSecPerRun == 0) { + results.result.cSpeed = 0; + } else { + results.result.cSpeed = (double)srcSize / compressionResults.result.nanoSecPerRun * TIMELOOP_NANOSEC; + } + results.result.cSize = compressionResults.result.sumOfReturn; + { + int const ratioAccuracy = (ratio < 10.) ? 3 : 2; + double const compressionSpeed = results.result.cSpeed / 1000000; + int const cSpeedAccuracy = (compressionSpeed < 10.) ? 2 : 1; + ratio = (double)srcSize / results.result.cSize; + markNb = (markNb+1) % NB_MARKS; + DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.*f),%6.*f MB/s\r", + marks[markNb], displayName, (U32)srcSize, (U32)results.result.cSize, + ratioAccuracy, ratio, + cSpeedAccuracy, compressionSpeed); + } } - results.result.cSize = compressionResults.result.sumOfReturn; - ratio = (double)srcSize / (double)results.result.cSize; - markNb = (markNb+1) % NB_MARKS; - { - int const ratioAccuracy = (ratio < 10.) ? 3 : 2; - double const compressionSpeed = ((double)srcSize / compressionResults.result.nanoSecPerRun) * 1000; - int const cSpeedAccuracy = (compressionSpeed < 10.) ? 2 : 1; - results.result.cSpeed = compressionSpeed * 1000000; - DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.*f),%6.*f MB/s\r", - marks[markNb], displayName, (U32)srcSize, (U32)results.result.cSize, - ratioAccuracy, ratio, - cSpeedAccuracy, compressionSpeed); - } } /* if (adv->mode != BMK_decodeOnly) */ if(adv->mode != BMK_compressOnly) { BMK_initDCtxArgs dctxprep; BMK_customReturn_t decompressionResults; - + U64 totalLoops = 0, totalTime = 0, fastest = (U64)(-1LL); + int completed = 0; + UTIL_time_t coolTime = UTIL_getTime(); dctxprep.dctx = dctx; dctxprep.dictBuffer = dictBuffer; dctxprep.dictBufferSize = dictBufferSize; - decompressionResults = BMK_benchFunction( - &local_defaultDecompress, (void*)(dctx), - &local_initDCtx, (void*)&dctxprep, - nbBlocks, - (const void * const *)cPtrs, cSizes, resPtrs, resSizes, - adv->loopMode, adv->nbSeconds); + if(adv->loopMode == BMK_timeMode) { + U64 maxTime = adv->nbSeconds * TIMELOOP_NANOSEC; + unsigned nbLoops = 1; + while(!completed) { + /* Overheat protection */ + if (UTIL_clockSpanMicro(coolTime) > ACTIVEPERIOD_MICROSEC) { + DEBUGOUTPUT("\rcooling down ... \r"); + UTIL_sleep(COOLPERIOD_SEC); + coolTime = UTIL_getTime(); + } + + decompressionResults = BMK_benchFunction( + &local_defaultDecompress, (void*)(dctx), + &local_initDCtx, (void*)&dctxprep, nbBlocks, + (const void * const *)cPtrs, cSizes, resPtrs, resSizes, + nbLoops); - if(decompressionResults.error) { - results.error = decompressionResults.error; - return results; - } + if(decompressionResults.error) { + results.error = decompressionResults.error; + return results; + } - markNb = (markNb+1) % NB_MARKS; - { int const ratioAccuracy = (ratio < 10.) ? 3 : 2; - double const compressionSpeed = results.result.cSpeed / 1000000; - int const cSpeedAccuracy = (compressionSpeed < 10.) ? 2 : 1; - double const decompressionSpeed = ((double)srcSize / decompressionResults.result.nanoSecPerRun) * 1000; - results.result.dSpeed = decompressionSpeed * 1000000; - DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.*f),%6.*f MB/s ,%6.1f MB/s \r", - marks[markNb], displayName, (U32)srcSize, (U32)results.result.cSize, - ratioAccuracy, ratio, - cSpeedAccuracy, compressionSpeed, - decompressionSpeed); + { U64 loopDuration = decompressionResults.result.nanoSecPerRun * nbLoops; + totalLoops += nbLoops; + totalTime += loopDuration; + if (loopDuration > 0) { + fastest = MIN(fastest, loopDuration / nbLoops); + nbLoops = (U32)(TIMELOOP_NANOSEC / fastest) + 1; + } else { + assert(nbLoops < 40000000); /* avoid overflow */ + nbLoops *= 2; + } + completed = (totalTime >= maxTime); + { int const ratioAccuracy = (ratio < 10.) ? 3 : 2; + double const compressionSpeed = results.result.cSpeed / 1000000; + int const cSpeedAccuracy = (compressionSpeed < 10.) ? 2 : 1; + double const decompressionSpeed = ((double)srcSize * totalLoops / totalTime) * 1000; + results.result.dSpeed = decompressionSpeed * 1000000; + markNb = (markNb+1) % NB_MARKS; + DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.*f),%6.*f MB/s ,%6.1f MB/s \r", + marks[markNb], displayName, (U32)srcSize, (U32)results.result.cSize, + ratioAccuracy, ratio, + cSpeedAccuracy, compressionSpeed, + decompressionSpeed); + } + } + } + } else { + decompressionResults = BMK_benchFunction( + &local_defaultDecompress, (void*)(dctx), + &local_initDCtx, (void*)&dctxprep, nbBlocks, + (const void * const *)cPtrs, cSizes, resPtrs, resSizes, + adv->nbSeconds); + if(decompressionResults.error) { + results.error = decompressionResults.error; + return results; + } + if(decompressionResults.result.nanoSecPerRun == 0) { + results.result.dSpeed = 0; + } else { + results.result.dSpeed = (double)srcSize / decompressionResults.result.nanoSecPerRun * TIMELOOP_NANOSEC; + } + { int const ratioAccuracy = (ratio < 10.) ? 3 : 2; + double const compressionSpeed = results.result.cSpeed / 1000000; + int const cSpeedAccuracy = (compressionSpeed < 10.) ? 2 : 1; + double const decompressionSpeed = ((double)srcSize / decompressionResults.result.nanoSecPerRun) * 1000; + results.result.dSpeed = decompressionSpeed * 1000000; + markNb = (markNb+1) % NB_MARKS; + DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.*f),%6.*f MB/s ,%6.1f MB/s \r", + marks[markNb], displayName, (U32)srcSize, (U32)results.result.cSize, + ratioAccuracy, ratio, + cSpeedAccuracy, compressionSpeed, + decompressionSpeed); + } } } @@ -622,7 +650,43 @@ BMK_return_t BMK_benchMemAdvanced(const void* srcBuffer, size_t srcSize, } DISPLAYLEVEL(2, "%2i#\n", cLevel); } /* Bench */ + return results; +} +BMK_return_t BMK_benchMemAdvanced(const void* srcBuffer, size_t srcSize, + const size_t* fileSizes, unsigned nbFiles, + const int cLevel, const ZSTD_compressionParameters* comprParams, + const void* dictBuffer, size_t dictBufferSize, + ZSTD_CCtx* ctx, ZSTD_DCtx* dctx, + int displayLevel, const char* displayName, const BMK_advancedParams_t* adv) + +{ + size_t const blockSize = ((adv->blockSize>=32 && (adv->mode != BMK_decodeOnly)) ? adv->blockSize : srcSize) + (!srcSize) /* avoid div by 0 */ ; + U32 const maxNbBlocks = (U32) ((srcSize + (blockSize-1)) / blockSize) + nbFiles; + + /* these are the blockTable parameters, just split up */ + const void ** const srcPtrs = (const void ** const)malloc(maxNbBlocks * sizeof(void*)); + size_t* const srcSizes = (size_t* const)malloc(maxNbBlocks * sizeof(size_t)); + + void ** const cPtrs = (void** const)malloc(maxNbBlocks * sizeof(void*)); + size_t* const cSizes = (size_t* const)malloc(maxNbBlocks * sizeof(size_t)); + + void ** const resPtrs = (void** const)malloc(maxNbBlocks * sizeof(void*)); + size_t* const resSizes = (size_t* const)malloc(maxNbBlocks * sizeof(size_t)); + + const size_t maxCompressedSize = ZSTD_compressBound(srcSize) + (maxNbBlocks * 1024); /* add some room for safety */ + void* compressedBuffer = malloc(maxCompressedSize); + void* resultBuffer = malloc(srcSize); + + BMK_return_t results; + int allocationincomplete = !compressedBuffer || !resultBuffer || + !srcPtrs || !srcSizes || !cPtrs || !cSizes || !resPtrs || !resSizes; + if (!allocationincomplete) { + results = BMK_benchMemAdvancedNoAlloc(srcPtrs, srcSizes, cPtrs, cSizes, + resPtrs, resSizes, resultBuffer, compressedBuffer, maxCompressedSize, + srcBuffer, srcSize, fileSizes, nbFiles, cLevel, comprParams, + dictBuffer, dictBufferSize, ctx, dctx, displayLevel, displayName, adv); + } /* clean up */ free(compressedBuffer); free(resultBuffer); @@ -634,6 +698,9 @@ BMK_return_t BMK_benchMemAdvanced(const void* srcBuffer, size_t srcSize, free(resPtrs); free(resSizes); + if(allocationincomplete) { + EXM_THROW(31, BMK_return_t, "allocation error : not enough memory"); + } results.error = 0; return results; } @@ -767,31 +834,43 @@ static int BMK_loadFiles(void* buffer, size_t bufferSize, return 0; } -static BMK_return_t BMK_benchFileTable(const char* const * const fileNamesTable, unsigned const nbFiles, +BMK_return_t BMK_benchFilesAdvanced(const char* const * const fileNamesTable, unsigned const nbFiles, const char* const dictFileName, int const cLevel, - const ZSTD_compressionParameters* const compressionParams, int displayLevel, - const BMK_advancedParams_t * const adv) + const ZSTD_compressionParameters* const compressionParams, + int displayLevel, const BMK_advancedParams_t * const adv) { void* srcBuffer; size_t benchedSize; void* dictBuffer = NULL; size_t dictBufferSize = 0; - size_t* const fileSizes = (size_t*)calloc(nbFiles, sizeof(size_t)); + size_t* fileSizes; BMK_return_t res; U64 const totalSizeToLoad = UTIL_getTotalFileSize(fileNamesTable, nbFiles); - if (!fileSizes) EXM_THROW(12, BMK_return_t, "not enough memory for fileSizes"); + if(!nbFiles) { + EXM_THROW(14, BMK_return_t, "No Files to Benchmark"); + } + if (cLevel > ZSTD_maxCLevel()) { + EXM_THROW(15, BMK_return_t, "Invalid Compression Level"); + } + + fileSizes = (size_t*)calloc(nbFiles, sizeof(size_t)); + if (!fileSizes) EXM_THROW(12, BMK_return_t, "not enough memory for fileSizes"); /* Load dictionary */ if (dictFileName != NULL) { U64 const dictFileSize = UTIL_getFileSize(dictFileName); - if (dictFileSize > 64 MB) + if (dictFileSize > 64 MB) { + free(fileSizes); EXM_THROW(10, BMK_return_t, "dictionary file %s too large", dictFileName); + } dictBufferSize = (size_t)dictFileSize; dictBuffer = malloc(dictBufferSize); - if (dictBuffer==NULL) + if (dictBuffer==NULL) { + free(fileSizes); EXM_THROW(11, BMK_return_t, "not enough memory for dictionary (%u bytes)", (U32)dictBufferSize); + } { int errorCode = BMK_loadFiles(dictBuffer, dictBufferSize, fileSizes, &dictFileName, 1, displayLevel); if(errorCode) { @@ -807,7 +886,11 @@ static BMK_return_t BMK_benchFileTable(const char* const * const fileNamesTable, if (benchedSize < totalSizeToLoad) DISPLAY("Not enough memory; testing %u MB only...\n", (U32)(benchedSize >> 20)); srcBuffer = malloc(benchedSize); - if (!srcBuffer) EXM_THROW(12, BMK_return_t, "not enough memory"); + if (!srcBuffer) { + free(dictBuffer); + free(fileSizes); + EXM_THROW(12, BMK_return_t, "not enough memory"); + } /* Load input buffer */ { @@ -839,15 +922,21 @@ static BMK_return_t BMK_benchFileTable(const char* const * const fileNamesTable, } -static BMK_return_t BMK_syntheticTest(int cLevel, double compressibility, +BMK_return_t BMK_syntheticTest(int cLevel, double compressibility, const ZSTD_compressionParameters* compressionParams, int displayLevel, const BMK_advancedParams_t * const adv) { char name[20] = {0}; size_t benchedSize = 10000000; - void* const srcBuffer = malloc(benchedSize); + void* srcBuffer; BMK_return_t res; + + if (cLevel > ZSTD_maxCLevel()) { + EXM_THROW(15, BMK_return_t, "Invalid Compression Level"); + } + /* Memory allocation */ + srcBuffer = malloc(benchedSize); if (!srcBuffer) EXM_THROW(21, BMK_return_t, "not enough memory"); /* Fill input buffer */ @@ -867,28 +956,9 @@ static BMK_return_t BMK_syntheticTest(int cLevel, double compressibility, return res; } - -BMK_return_t BMK_benchFilesAdvanced(const char** fileNamesTable, unsigned nbFiles, - const char* dictFileName, - int cLevel, const ZSTD_compressionParameters* compressionParams, - int displayLevel, const BMK_advancedParams_t * const adv) -{ - double const compressibility = (double)g_compressibilityDefault / 100; - - if (cLevel > ZSTD_maxCLevel()) { - EXM_THROW(15, BMK_return_t, "Invalid Compression Level"); - } - if (nbFiles == 0) { - return BMK_syntheticTest(cLevel, compressibility, compressionParams, displayLevel, adv); - } - else { - return BMK_benchFileTable(fileNamesTable, nbFiles, dictFileName, cLevel, compressionParams, displayLevel, adv); - } -} - -BMK_return_t BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles, - const char* dictFileName, - int cLevel, const ZSTD_compressionParameters* compressionParams, +BMK_return_t BMK_benchFiles(const char* const * const fileNamesTable, unsigned const nbFiles, + const char* const dictFileName, + int const cLevel, const ZSTD_compressionParameters* const compressionParams, int displayLevel) { const BMK_advancedParams_t adv = BMK_initAdvancedParams(); return BMK_benchFilesAdvanced(fileNamesTable, nbFiles, dictFileName, cLevel, compressionParams, displayLevel, &adv); diff --git a/programs/bench.h b/programs/bench.h index d553e7b4f..5f3a55285 100644 --- a/programs/bench.h +++ b/programs/bench.h @@ -40,7 +40,8 @@ typedef struct { size_t sumOfReturn; /* sum of return values */ U64 nanoSecPerRun; /* time per iteration */ } BMK_customResult_t; - +//we might need a nbRuns or nbSecs if we're keeping timeMode / iterMode respectively. +//give benchMem responsibility to incrementally update display. ERROR_STRUCT(BMK_result_t, BMK_return_t); ERROR_STRUCT(BMK_customResult_t, BMK_customReturn_t); @@ -78,7 +79,7 @@ BMK_advancedParams_t BMK_initAdvancedParams(void); /* Loads files in fileNamesTable into memory, as well as a dictionary * from dictFileName, and then uses benchMem */ /* fileNamesTable - name of files to benchmark - * nbFiles - number of files (size of fileNamesTable) + * nbFiles - number of files (size of fileNamesTable), must be > 0 * dictFileName - name of dictionary file to load * cLevel - compression level to benchmark, errors if invalid * compressionParams - basic compression Parameters @@ -91,19 +92,37 @@ BMK_advancedParams_t BMK_initAdvancedParams(void); * return * .error will give a nonzero error value if an error has occured * .result - if .error = 0, .result will return the time taken to compression speed - * (.cSpeed), decompression speed (.dSpeed), and copmressed size (.cSize) of the original + * (.cSpeed), decompression speed (.dSpeed), and compressed size (.cSize) of the original * file */ -BMK_return_t BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles, const char* dictFileName, - int cLevel, const ZSTD_compressionParameters* compressionParams, +BMK_return_t BMK_benchFiles(const char* const * const fileNamesTable, unsigned const nbFiles, + const char* const dictFileName, + int const cLevel, const ZSTD_compressionParameters* const compressionParams, int displayLevel); /* See benchFiles for normal parameter uses and return, see advancedParams_t for adv */ -BMK_return_t BMK_benchFilesAdvanced(const char** fileNamesTable, unsigned nbFiles, - const char* dictFileName, - int cLevel, const ZSTD_compressionParameters* compressionParams, +BMK_return_t BMK_benchFilesAdvanced(const char* const * const fileNamesTable, unsigned const nbFiles, + const char* const dictFileName, + int const cLevel, const ZSTD_compressionParameters* const compressionParams, int displayLevel, const BMK_advancedParams_t* const adv); +/* called in cli */ +/* Generates a sample with datagen with the compressibility argument*/ +/* cLevel - compression level to benchmark, errors if invalid + * compressibility - determines compressibility of sample + * compressionParams - basic compression Parameters + * displayLevel - see benchFiles + * adv - see advanced_Params_t + * return + * .error will give a nonzero error value if an error has occured + * .result - if .error = 0, .result will return the time taken to compression speed + * (.cSpeed), decompression speed (.dSpeed), and compressed size (.cSize) of the original + * file + */ +BMK_return_t BMK_syntheticTest(int cLevel, double compressibility, + const ZSTD_compressionParameters* compressionParams, + int displayLevel, const BMK_advancedParams_t * const adv); + /* basic benchmarking function, called in paramgrill * applies ZSTD_compress_generic() and ZSTD_decompress_generic() on data in srcBuffer * with specific compression parameters specified by other arguments using benchFunction @@ -150,8 +169,7 @@ BMK_return_t BMK_benchMemAdvanced(const void* srcBuffer, size_t srcSize, * srcSizes - an array of the sizes of above buffers * dstBuffers - an array of buffers to be written into by benchFn * dstCapacities - an array of the capacities of above buffers. - * mode - if 0, iter will be interpreted as the minimum number of seconds to run - * iter - see mode + * iter - defines number of times benchFn is run. * return * .error will give a nonzero value if ZSTD_isError() is nonzero for any of the return * of the calls to initFn and benchFn, or if benchFunction errors internally @@ -168,7 +186,7 @@ BMK_customReturn_t BMK_benchFunction( size_t blockCount, const void* const * const srcBuffers, const size_t* srcSizes, void* const * const dstBuffers, const size_t* dstCapacities, - unsigned mode, unsigned iter); + unsigned sec); #endif /* BENCH_H_121279284357 */ diff --git a/programs/zstdcli.c b/programs/zstdcli.c index 63be9ef61..a450ecac2 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -398,6 +398,7 @@ int main(int argCount, const char* argv[]) setRealTimePrio = 0, singleThread = 0, ultra=0; + double compressibility = 0.5; BMK_advancedParams_t adv = BMK_initAdvancedParams(); unsigned bench_nbSeconds = 3; /* would be better if this value was synchronized from bench */ size_t blockSize = 0; @@ -706,6 +707,19 @@ int main(int argCount, const char* argv[]) #endif main_pause=1; break; + + /* Select compressibility of synthetic sample */ + case 'P': + { U32 proba32 = 0; + while ((argument[1]>= '0') && (argument[1]<= '9')) { + proba32 *= 10; + proba32 += argument[1] - '0'; + argument++; + } + compressibility = (double)proba32 / 100; + } + break; + /* unknown command */ default : CLEAN_RETURN(badusage(programName)); } @@ -821,20 +835,25 @@ int main(int argCount, const char* argv[]) if (cLevelLast < cLevel) cLevelLast = cLevel; if (cLevelLast > cLevel) DISPLAYLEVEL(2, "Benchmarking levels from %d to %d\n", cLevel, cLevelLast); - - if(separateFiles) { - unsigned i; - for(i = 0; i < filenameIdx; i++) { - DISPLAYLEVEL(2, "Benchmarking %s \n", filenameTable[i]); - int c; - for(c = cLevel; c <= cLevelLast; c++) { - BMK_benchFilesAdvanced(&filenameTable[i], 1, dictFileName, c, &compressionParams, g_displayLevel, &adv); + if(filenameIdx) { + if(separateFiles) { + unsigned i; + for(i = 0; i < filenameIdx; i++) { + int c; + DISPLAYLEVEL(2, "Benchmarking %s \n", filenameTable[i]); + for(c = cLevel; c <= cLevelLast; c++) { + BMK_benchFilesAdvanced(&filenameTable[i], 1, dictFileName, c, &compressionParams, g_displayLevel, &adv); + } } + } else { + for(; cLevel <= cLevelLast; cLevel++) { + BMK_benchFilesAdvanced(filenameTable, filenameIdx, dictFileName, cLevel, &compressionParams, g_displayLevel, &adv); + } } } else { for(; cLevel <= cLevelLast; cLevel++) { - BMK_benchFilesAdvanced(filenameTable, filenameIdx, dictFileName, cLevel, &compressionParams, g_displayLevel, &adv); - } + BMK_syntheticTest(cLevel, compressibility, &compressionParams, g_displayLevel, &adv); + } } #else diff --git a/tests/fullbench.c b/tests/fullbench.c index 91a1370df..0a18eb2d9 100644 --- a/tests/fullbench.c +++ b/tests/fullbench.c @@ -291,6 +291,8 @@ static size_t benchMem(const void* src, size_t srcSize, U32 benchNb) void* buff2; const char* benchName; size_t (*benchFunction)(const void* src, size_t srcSize, void* dst, size_t dstSize, void* verifBuff); + BMK_customReturn_t r; + int errorcode = 0; /* Selection */ switch(benchNb) @@ -427,15 +429,13 @@ static size_t benchMem(const void* src, size_t srcSize, U32 benchNb) /* benchmark loop */ { - BMK_customReturn_t r = BMK_benchFunction( - benchFunction, buff2, - NULL, NULL, - 1, &src, &srcSize, - (void * const * const)&dstBuff, &dstBuffSize, - BMK_timeMode, 1); + r = BMK_benchFunction(benchFunction, buff2, + NULL, NULL, 1, &src, &srcSize, + (void * const * const)&dstBuff, &dstBuffSize, g_nbIterations); if(r.error) { DISPLAY("ERROR %d ! ! \n", r.error); - exit(1); + errorcode = r.error; + goto _cleanOut; } DISPLAY("%2u#Speed: %f MB/s - Size: %f MB - %s\n", benchNb, (double)srcSize / r.result.nanoSecPerRun * 1000, (double)r.result.sumOfReturn / 1000000, benchName); @@ -448,7 +448,7 @@ _cleanOut: ZSTD_freeDCtx(g_zdc); g_zdc=NULL; ZSTD_freeCStream(g_cstream); g_cstream=NULL; ZSTD_freeDStream(g_dstream); g_dstream=NULL; - return 0; + return errorcode; } From ab26f24c9c8ec8e6435c09d8c69b1a8faa19b852 Mon Sep 17 00:00:00 2001 From: George Lu Date: Thu, 21 Jun 2018 11:16:53 -0700 Subject: [PATCH 008/372] benchFunction Timed Wrappers Add BMK_benchFunctionTimed Add BMK_init_customResultCont.. Change benchMem to use benchFunctionTimed Minor Fixes/Adjustments --- programs/bench.c | 185 +++++++++++++++++++++++++---------------------- programs/bench.h | 34 ++++++++- 2 files changed, 133 insertions(+), 86 deletions(-) diff --git a/programs/bench.c b/programs/bench.c index 447f9feb9..9aa486f8d 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -333,6 +333,70 @@ BMK_customReturn_t BMK_benchFunction( retval.result.nanoSecPerRun = totalTime / iter; retval.result.sumOfReturn = dstSize; return retval; +} + +BMK_customResultContinuation_t BMK_init_customResultContinuation(unsigned iter) { + BMK_customResultContinuation_t c; + c.completed = 0; + c.state.nbLoops = 1; + c.state.coolTime = UTIL_getTime(); + c.state.timeRemaining = (U64)iter * TIMELOOP_NANOSEC; + c.intermediateResult.error = 0; + c.intermediateResult.result.nanoSecPerRun = (U64)(-1LL); + c.intermediateResult.result.sumOfReturn = 0; + return c; +} + +#define MINUSABLETIME 500000000ULL + +//how to use minusabletime? +//only report times which are > minUsable +void BMK_benchFunctionTimed( + size_t (*benchFn)(const void*, size_t, void*, size_t, void*), void* benchPayload, + size_t (*initFn)(void*), void* initPayload, + size_t blockCount, + const void* const * const srcBlockBuffers, const size_t* srcBlockSizes, + void* const * const dstBlockBuffers, const size_t* dstBlockCapacities, + BMK_customResultContinuation_t* cont) +{ + U64 fastest = cont->intermediateResult.result.nanoSecPerRun; + int completed = 0; + + while(!cont->completed && !completed) + { + /* Overheat protection */ + if (UTIL_clockSpanMicro(cont->state.coolTime) > ACTIVEPERIOD_MICROSEC) { + DEBUGOUTPUT("\rcooling down ... \r"); + UTIL_sleep(COOLPERIOD_SEC); + cont->state.coolTime = UTIL_getTime(); + } + + cont->intermediateResult = BMK_benchFunction(benchFn, benchPayload, initFn, initPayload, + blockCount, srcBlockBuffers, srcBlockSizes, dstBlockBuffers, dstBlockCapacities, cont->state.nbLoops); + if(cont->intermediateResult.error) { /* completed w/ error */ + cont->completed = 1; + return; + } + + { U64 const loopDuration = cont->intermediateResult.result.nanoSecPerRun * cont->state.nbLoops; + cont->completed = (cont->state.timeRemaining <= loopDuration); + cont->state.timeRemaining -= loopDuration; + if (loopDuration > 0) { + fastest = MIN(fastest, cont->intermediateResult.result.nanoSecPerRun); + cont->intermediateResult.result.nanoSecPerRun = fastest; + cont->state.nbLoops = (U32)(TIMELOOP_NANOSEC / fastest) + 1; + } else { + const unsigned multiplier = 2; + assert(cont->state.nbLoops < ((unsigned)-1) / multiplier); /* avoid overflow */ + cont->state.nbLoops *= multiplier; + } + if(loopDuration < MINUSABLETIME) { /* don't report results which have time too low */ + continue; + } + + } + completed = 1; + } } /* benchMem with no allocation */ @@ -350,7 +414,7 @@ static BMK_return_t BMK_benchMemAdvancedNoAlloc( ZSTD_CCtx* ctx, ZSTD_DCtx* dctx, int displayLevel, const char* displayName, const BMK_advancedParams_t* adv) { - size_t const blockSize = ((adv->blockSize>=32 && (adv->mode != BMK_decodeOnly)) ? adv->blockSize : srcSize) + (!srcSize) /* avoid div by 0 */ ; + size_t const blockSize = ((adv->blockSize>=32 && (adv->mode != BMK_decodeOnly)) ? adv->blockSize : srcSize) + (!srcSize); /* avoid div by 0 */ BMK_return_t results; size_t const loadedCompressedSize = srcSize; size_t cSize = 0; @@ -428,12 +492,9 @@ static BMK_return_t BMK_benchMemAdvancedNoAlloc( U32 markNb = 0; DISPLAYLEVEL(2, "\r%79s\r", ""); + if (adv->mode != BMK_decodeOnly) { BMK_initCCtxArgs cctxprep; - BMK_customReturn_t compressionResults; - int completed = 0; - U64 totalLoops = 0, totalTime = 0, fastest = (U64)(-1LL); - UTIL_time_t coolTime = UTIL_getTime(); cctxprep.ctx = ctx; cctxprep.dictBuffer = dictBuffer; cctxprep.dictBufferSize = dictBufferSize; @@ -443,51 +504,31 @@ static BMK_return_t BMK_benchMemAdvancedNoAlloc( /* Compression */ DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->\r", marks[markNb], displayName, (U32)srcSize); if(adv->loopMode == BMK_timeMode) { - U64 maxTime = adv->nbSeconds * TIMELOOP_NANOSEC; - unsigned nbLoops = 1; - while(!completed) { - /* Overheat protection */ - if (UTIL_clockSpanMicro(coolTime) > ACTIVEPERIOD_MICROSEC) { - DEBUGOUTPUT("\rcooling down ... \r"); - UTIL_sleep(COOLPERIOD_SEC); - coolTime = UTIL_getTime(); - } - - compressionResults = BMK_benchFunction(&local_defaultCompress, (void*)ctx, &local_initCCtx, (void*)&cctxprep, - nbBlocks, srcPtrs, srcSizes, cPtrs, cSizes, nbLoops); - if(compressionResults.error) { - results.error = compressionResults.error; + BMK_customResultContinuation_t cont = BMK_init_customResultContinuation(adv->nbSeconds); + while(!cont.completed) { + BMK_benchFunctionTimed(&local_defaultCompress, (void*)ctx, &local_initCCtx, (void*)&cctxprep, + nbBlocks, srcPtrs, srcSizes, cPtrs, cSizes, &cont); + if(cont.intermediateResult.error) { + results.error = cont.intermediateResult.error; return results; } - - { U64 loopDuration = compressionResults.result.nanoSecPerRun * nbLoops; - totalLoops += nbLoops; - totalTime += loopDuration; - if (loopDuration > 0) { // nanoSec / run - fastest = MIN(fastest, compressionResults.result.nanoSecPerRun); - nbLoops = (U32)(TIMELOOP_NANOSEC / fastest) + 1; - } else { - assert(nbLoops < 40000000); /* avoid overflow */ - nbLoops *= 2; - } - completed = (totalTime >= maxTime); - { - int const ratioAccuracy = (ratio < 10.) ? 3 : 2; - double const compressionSpeed = (((double)srcSize * totalLoops) / totalTime) * 1000; - int const cSpeedAccuracy = (compressionSpeed < 10.) ? 2 : 1; - results.result.cSpeed = compressionSpeed * 1000000; - results.result.cSize = compressionResults.result.sumOfReturn; - ratio = (double)srcSize / results.result.cSize; - markNb = (markNb+1) % NB_MARKS; - DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.*f),%6.*f MB/s\r", - marks[markNb], displayName, (U32)srcSize, (U32)results.result.cSize, - ratioAccuracy, ratio, - cSpeedAccuracy, compressionSpeed); - } - } + ratio = (double)(srcSize / cont.intermediateResult.result.sumOfReturn); + { + int const ratioAccuracy = (ratio < 10.) ? 3 : 2; + double const compressionSpeed = ((double)srcSize / cont.intermediateResult.result.nanoSecPerRun) * 1000; + int const cSpeedAccuracy = (compressionSpeed < 10.) ? 2 : 1; + results.result.cSpeed = compressionSpeed * 1000000; + results.result.cSize = cont.intermediateResult.result.sumOfReturn; + ratio = (double)srcSize / results.result.cSize; + markNb = (markNb+1) % NB_MARKS; + DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.*f),%6.*f MB/s\r", + marks[markNb], displayName, (U32)srcSize, (U32)results.result.cSize, + ratioAccuracy, ratio, + cSpeedAccuracy, compressionSpeed); + } } } else { - compressionResults = BMK_benchFunction(&local_defaultCompress, (void*)ctx, &local_initCCtx, (void*)&cctxprep, + BMK_customReturn_t compressionResults = BMK_benchFunction(&local_defaultCompress, (void*)ctx, &local_initCCtx, (void*)&cctxprep, nbBlocks, srcPtrs, srcSizes, cPtrs, cSizes, adv->nbSeconds); if(compressionResults.error) { results.error = compressionResults.error; @@ -517,49 +558,23 @@ static BMK_return_t BMK_benchMemAdvancedNoAlloc( if(adv->mode != BMK_compressOnly) { BMK_initDCtxArgs dctxprep; BMK_customReturn_t decompressionResults; - U64 totalLoops = 0, totalTime = 0, fastest = (U64)(-1LL); - int completed = 0; - UTIL_time_t coolTime = UTIL_getTime(); dctxprep.dctx = dctx; dctxprep.dictBuffer = dictBuffer; dctxprep.dictBufferSize = dictBufferSize; if(adv->loopMode == BMK_timeMode) { - U64 maxTime = adv->nbSeconds * TIMELOOP_NANOSEC; - unsigned nbLoops = 1; - while(!completed) { - /* Overheat protection */ - if (UTIL_clockSpanMicro(coolTime) > ACTIVEPERIOD_MICROSEC) { - DEBUGOUTPUT("\rcooling down ... \r"); - UTIL_sleep(COOLPERIOD_SEC); - coolTime = UTIL_getTime(); - } - - decompressionResults = BMK_benchFunction( - &local_defaultDecompress, (void*)(dctx), - &local_initDCtx, (void*)&dctxprep, nbBlocks, - (const void * const *)cPtrs, cSizes, resPtrs, resSizes, - nbLoops); - - if(decompressionResults.error) { - results.error = decompressionResults.error; + BMK_customResultContinuation_t cont = BMK_init_customResultContinuation(adv->nbSeconds); + while(!cont.completed) { + BMK_benchFunctionTimed(&local_defaultDecompress, (void*)(dctx), &local_initDCtx, (void*)&dctxprep, + nbBlocks, (const void * const *)cPtrs, cSizes, resPtrs, resSizes, &cont); + if(cont.intermediateResult.error) { + results.error = cont.intermediateResult.error; return results; } - - { U64 loopDuration = decompressionResults.result.nanoSecPerRun * nbLoops; - totalLoops += nbLoops; - totalTime += loopDuration; - if (loopDuration > 0) { - fastest = MIN(fastest, loopDuration / nbLoops); - nbLoops = (U32)(TIMELOOP_NANOSEC / fastest) + 1; - } else { - assert(nbLoops < 40000000); /* avoid overflow */ - nbLoops *= 2; - } - completed = (totalTime >= maxTime); - { int const ratioAccuracy = (ratio < 10.) ? 3 : 2; + + { int const ratioAccuracy = (ratio < 10.) ? 3 : 2; double const compressionSpeed = results.result.cSpeed / 1000000; int const cSpeedAccuracy = (compressionSpeed < 10.) ? 2 : 1; - double const decompressionSpeed = ((double)srcSize * totalLoops / totalTime) * 1000; + double const decompressionSpeed = ((double)srcSize / cont.intermediateResult.result.nanoSecPerRun) * 1000; results.result.dSpeed = decompressionSpeed * 1000000; markNb = (markNb+1) % NB_MARKS; DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.*f),%6.*f MB/s ,%6.1f MB/s \r", @@ -567,8 +582,7 @@ static BMK_return_t BMK_benchMemAdvancedNoAlloc( ratioAccuracy, ratio, cSpeedAccuracy, compressionSpeed, decompressionSpeed); - } - } + } } } else { decompressionResults = BMK_benchFunction( @@ -643,10 +657,11 @@ static BMK_return_t BMK_benchMemAdvancedNoAlloc( if (displayLevel == 1) { /* hidden display mode -q, used by python speed benchmark */ double const cSpeed = results.result.cSpeed / 1000000; double const dSpeed = results.result.dSpeed / 1000000; - if (adv->additionalParam) + if (adv->additionalParam) { DISPLAY("-%-3i%11i (%5.3f) %6.2f MB/s %6.1f MB/s %s (param=%d)\n", cLevel, (int)cSize, ratio, cSpeed, dSpeed, displayName, adv->additionalParam); - else + } else { DISPLAY("-%-3i%11i (%5.3f) %6.2f MB/s %6.1f MB/s %s\n", cLevel, (int)cSize, ratio, cSpeed, dSpeed, displayName); + } } DISPLAYLEVEL(2, "%2i#\n", cLevel); } /* Bench */ diff --git a/programs/bench.h b/programs/bench.h index 5f3a55285..17cc0d0f5 100644 --- a/programs/bench.h +++ b/programs/bench.h @@ -161,7 +161,7 @@ BMK_return_t BMK_benchMemAdvanced(const void* srcBuffer, size_t srcSize, /* This function times the execution of 2 argument functions, benchFn and initFn */ /* benchFn - (*benchFn)(srcBuffers[i], srcSizes[i], dstBuffers[i], dstCapacities[i], benchPayload) - * is run a variable number of times, specified by mode and iter args + * is run iter times * initFn - (*initFn)(initPayload) is run once per benchmark at the beginning. This argument can * be NULL, in which case nothing is run. * blockCount - number of blocks (size of srcBuffers, srcSizes, dstBuffers, dstCapacities) @@ -188,6 +188,38 @@ BMK_customReturn_t BMK_benchFunction( void* const * const dstBuffers, const size_t* dstCapacities, unsigned sec); +typedef struct { + unsigned nbLoops; + U64 timeRemaining; + UTIL_time_t coolTime; +} BMK_timeState_t; + +typedef struct { + int completed; + BMK_customReturn_t intermediateResult; /* since the wrapper can't err, don't need ERROR_STRUCT(cRC, just check here) */ + BMK_timeState_t state; +} BMK_customResultContinuation_t; + +/* + * initializes the last argument of benchFunctionTimed, with iter being the number of seconds to bench (see below) + */ +BMK_customResultContinuation_t BMK_init_customResultContinuation(unsigned iter); + +/* + * Benchmarks custom functions like BMK_benchFunction(), but runs for iter seconds rather than a fixed number of iterations + * arguments mostly the same other than BMK_benchFunction() + * Usage - benchFunctionTimed will return in approximately one second, where the intermediate results can be found in + * the *cont passed in and be displayed/used as wanted. Keep calling BMK_benchFunctionTimed() until cont->completed = 1 + * to continue updating intermediate result. + */ +void BMK_benchFunctionTimed( + size_t (*benchFn)(const void*, size_t, void*, size_t, void*), void* benchPayload, + size_t (*initFn)(void*), void* initPayload, + size_t blockCount, + const void* const * const srcBlockBuffers, const size_t* srcBlockSizes, + void* const * const dstBlockBuffers, const size_t* dstBlockCapacities, + BMK_customResultContinuation_t* cont); + #endif /* BENCH_H_121279284357 */ #if defined (__cplusplus) From d6121ad0e1037de73817b9c8568f5dbbe0c22925 Mon Sep 17 00:00:00 2001 From: George Lu Date: Fri, 22 Jun 2018 17:25:16 -0700 Subject: [PATCH 009/372] Opaque State And minor fixups (comments/alignment/checks/fix memory leak) --- programs/bench.c | 207 +++++++++++++++++++++++++-------------------- programs/bench.h | 128 ++++++++++++++-------------- programs/zstdcli.c | 9 +- tests/fullbench.c | 2 +- 4 files changed, 181 insertions(+), 165 deletions(-) diff --git a/programs/bench.c b/programs/bench.c index 9aa486f8d..b2b5dc3b2 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -42,6 +42,7 @@ #include "datagen.h" /* RDG_genBuffer */ #include "xxhash.h" #include "bench.h" +#include "zstd_errors.h" /* ************************************* @@ -108,13 +109,13 @@ static UTIL_time_t g_displayClock = UTIL_TIME_INITIALIZER; } /* error without displaying */ -#define EXM_THROW_ND(errorNum, retType, ...) { \ +#define EXM_THROW_ND(errorNum, retType, ...) { \ retType r; \ memset(&r, 0, sizeof(retType)); \ DEBUGOUTPUT("%s: %i: \n", __FILE__, __LINE__); \ - DEBUGOUTPUT("Error %i : ", errorNum); \ - DEBUGOUTPUT(__VA_ARGS__); \ - DEBUGOUTPUT(" \n"); \ + DEBUGOUTPUT("Error %i : ", errorNum); \ + DEBUGOUTPUT(__VA_ARGS__); \ + DEBUGOUTPUT(" \n"); \ r.error = errorNum; \ return r; \ } @@ -123,8 +124,6 @@ static UTIL_time_t g_displayClock = UTIL_TIME_INITIALIZER; * Benchmark Parameters ***************************************/ -#define BMK_LDM_PARAM_NOTSET 9999 - BMK_advancedParams_t BMK_initAdvancedParams(void) { BMK_advancedParams_t res = { BMK_both, /* mode */ @@ -137,8 +136,8 @@ BMK_advancedParams_t BMK_initAdvancedParams(void) { 0, /* ldmFlag */ 0, /* ldmMinMatch */ 0, /* ldmHashLog */ - BMK_LDM_PARAM_NOTSET, /* ldmBuckSizeLog */ - BMK_LDM_PARAM_NOTSET /* ldmHashEveryLog */ + 0, /* ldmBuckSizeLog */ + 0 /* ldmHashEveryLog */ }; return res; } @@ -157,6 +156,13 @@ typedef struct { size_t resSize; } blockParam_t; +struct BMK_timeState_t{ + unsigned nbLoops; + U64 timeRemaining; + UTIL_time_t coolTime; + U64 fastestTime; +}; + #undef MIN #undef MAX #define MIN(a,b) ((a) < (b) ? (a) : (b)) @@ -174,12 +180,8 @@ static void BMK_initCCtx(ZSTD_CCtx* ctx, ZSTD_CCtx_setParameter(ctx, ZSTD_p_enableLongDistanceMatching, adv->ldmFlag); ZSTD_CCtx_setParameter(ctx, ZSTD_p_ldmMinMatch, adv->ldmMinMatch); ZSTD_CCtx_setParameter(ctx, ZSTD_p_ldmHashLog, adv->ldmHashLog); - if (adv->ldmBucketSizeLog != BMK_LDM_PARAM_NOTSET) { - ZSTD_CCtx_setParameter(ctx, ZSTD_p_ldmBucketSizeLog, adv->ldmBucketSizeLog); - } - if (adv->ldmHashEveryLog != BMK_LDM_PARAM_NOTSET) { - ZSTD_CCtx_setParameter(ctx, ZSTD_p_ldmHashEveryLog, adv->ldmHashEveryLog); - } + ZSTD_CCtx_setParameter(ctx, ZSTD_p_ldmBucketSizeLog, adv->ldmBucketSizeLog); + ZSTD_CCtx_setParameter(ctx, ZSTD_p_ldmHashEveryLog, adv->ldmHashEveryLog); ZSTD_CCtx_setParameter(ctx, ZSTD_p_windowLog, comprParams->windowLog); ZSTD_CCtx_setParameter(ctx, ZSTD_p_hashLog, comprParams->hashLog); ZSTD_CCtx_setParameter(ctx, ZSTD_p_chainLog, comprParams->chainLog); @@ -239,6 +241,9 @@ static size_t local_defaultCompress( out.size = dstSize; out.pos = 0; while (moreToFlush) { + if(out.pos == out.size) { + return (size_t)-ZSTD_error_dstSize_tooSmall; + } moreToFlush = ZSTD_compress_generic(ctx, &out, &in, ZSTD_e_end); if (ZSTD_isError(moreToFlush)) { return moreToFlush; @@ -263,6 +268,9 @@ static size_t local_defaultDecompress( out.size = dstSize; out.pos = 0; while (moreToFlush) { + if(out.pos == out.size) { + return (size_t)-ZSTD_error_dstSize_tooSmall; + } moreToFlush = ZSTD_decompress_generic(dctx, &out, &in); if (ZSTD_isError(moreToFlush)) { @@ -276,17 +284,16 @@ static size_t local_defaultDecompress( /* initFn will be measured once, bench fn will be measured x times */ /* benchFn should return error value or out Size */ /* takes # of blocks and list of size & stuff for each. */ -/* only does iterations*/ -/* note time/iter could be zero if interval too short */ +/* only does looping */ +/* note time per loop could be zero if interval too short */ BMK_customReturn_t BMK_benchFunction( - size_t (*benchFn)(const void*, size_t, void*, size_t, void*), void* benchPayload, - size_t (*initFn)(void*), void* initPayload, + BMK_benchFn_t benchFn, void* benchPayload, + BMK_initFn_t initFn, void* initPayload, size_t blockCount, const void* const * const srcBlockBuffers, const size_t* srcBlockSizes, void* const * const dstBlockBuffers, const size_t* dstBlockCapacities, - unsigned iter) { + unsigned nbLoops) { size_t srcSize = 0, dstSize = 0, ind = 0; - unsigned toAdd = 1; U64 totalTime; BMK_customReturn_t retval; @@ -302,7 +309,7 @@ BMK_customReturn_t BMK_benchFunction( UTIL_waitForNextTick(); } - if(!iter) { + if(!nbLoops) { EXM_THROW_ND(1, BMK_customReturn_t, "nbLoops must be nonzero \n"); } @@ -310,85 +317,93 @@ BMK_customReturn_t BMK_benchFunction( srcSize += srcBlockSizes[ind]; } - { - unsigned i, j; + { + unsigned i, j, firstIter = 1; clockStart = UTIL_getTime(); - if(initFn != NULL) { (*initFn)(initPayload); } - for(i = 0; i < iter; i++) { + if(initFn != NULL) { initFn(initPayload); } + for(i = 0; i < nbLoops; i++) { for(j = 0; j < blockCount; j++) { - size_t res = (*benchFn)(srcBlockBuffers[j], srcBlockSizes[j], dstBlockBuffers[j], dstBlockCapacities[j], benchPayload); + size_t res = benchFn(srcBlockBuffers[j], srcBlockSizes[j], dstBlockBuffers[j], dstBlockCapacities[j], benchPayload); if(ZSTD_isError(res)) { EXM_THROW_ND(2, BMK_customReturn_t, "Function benchmarking failed on block %u of size %u : %s \n", j, (U32)dstBlockCapacities[j], ZSTD_getErrorName(res)); - } else if(toAdd) { + } else if(firstIter) { dstSize += res; } } - toAdd = 0; + firstIter = 0; } totalTime = UTIL_clockSpanNano(clockStart); } retval.error = 0; - retval.result.nanoSecPerRun = totalTime / iter; + retval.result.nanoSecPerRun = totalTime / nbLoops; retval.result.sumOfReturn = dstSize; return retval; } -BMK_customResultContinuation_t BMK_init_customResultContinuation(unsigned iter) { - BMK_customResultContinuation_t c; - c.completed = 0; - c.state.nbLoops = 1; - c.state.coolTime = UTIL_getTime(); - c.state.timeRemaining = (U64)iter * TIMELOOP_NANOSEC; - c.intermediateResult.error = 0; - c.intermediateResult.result.nanoSecPerRun = (U64)(-1LL); - c.intermediateResult.result.sumOfReturn = 0; - return c; +#define MINUSABLETIME 500000000ULL /* 0.5 seconds in ns */ + +void BMK_resetTimeState(BMK_timedFnState_t* r, unsigned nbSeconds) { + r->nbLoops = 1; + r->timeRemaining = (U64)nbSeconds * TIMELOOP_NANOSEC; + r->coolTime = UTIL_getTime(); + r->fastestTime = (U64)(-1LL); } -#define MINUSABLETIME 500000000ULL +BMK_timedFnState_t* BMK_createTimeState(unsigned nbSeconds) { + BMK_timedFnState_t* r = (BMK_timedFnState_t*)malloc(sizeof(struct BMK_timeState_t)); + BMK_resetTimeState(r, nbSeconds); + return r; +} -//how to use minusabletime? -//only report times which are > minUsable -void BMK_benchFunctionTimed( - size_t (*benchFn)(const void*, size_t, void*, size_t, void*), void* benchPayload, - size_t (*initFn)(void*), void* initPayload, +void BMK_freeTimeState(BMK_timedFnState_t* state) { + free(state); +} + +BMK_customTimedReturn_t BMK_benchFunctionTimed( + BMK_timedFnState_t* cont, + BMK_benchFn_t benchFn, void* benchPayload, + BMK_initFn_t initFn, void* initPayload, size_t blockCount, - const void* const * const srcBlockBuffers, const size_t* srcBlockSizes, - void* const * const dstBlockBuffers, const size_t* dstBlockCapacities, - BMK_customResultContinuation_t* cont) + const void* const* const srcBlockBuffers, const size_t* srcBlockSizes, + void* const* const dstBlockBuffers, const size_t* dstBlockCapacities) { - U64 fastest = cont->intermediateResult.result.nanoSecPerRun; + U64 fastest = cont->fastestTime; int completed = 0; + BMK_customTimedReturn_t r; + r.completed = 0; - while(!cont->completed && !completed) + while(!r.completed && !completed) { /* Overheat protection */ - if (UTIL_clockSpanMicro(cont->state.coolTime) > ACTIVEPERIOD_MICROSEC) { + if (UTIL_clockSpanMicro(cont->coolTime) > ACTIVEPERIOD_MICROSEC) { DEBUGOUTPUT("\rcooling down ... \r"); UTIL_sleep(COOLPERIOD_SEC); - cont->state.coolTime = UTIL_getTime(); + cont->coolTime = UTIL_getTime(); } - cont->intermediateResult = BMK_benchFunction(benchFn, benchPayload, initFn, initPayload, - blockCount, srcBlockBuffers, srcBlockSizes, dstBlockBuffers, dstBlockCapacities, cont->state.nbLoops); - if(cont->intermediateResult.error) { /* completed w/ error */ - cont->completed = 1; - return; + r.result = BMK_benchFunction(benchFn, benchPayload, initFn, initPayload, + blockCount, srcBlockBuffers, srcBlockSizes, dstBlockBuffers, dstBlockCapacities, cont->nbLoops); + if(r.result.error) { /* completed w/ error */ + r.completed = 1; + return r; } - { U64 const loopDuration = cont->intermediateResult.result.nanoSecPerRun * cont->state.nbLoops; - cont->completed = (cont->state.timeRemaining <= loopDuration); - cont->state.timeRemaining -= loopDuration; + { U64 const loopDuration = r.result.result.nanoSecPerRun * cont->nbLoops; + r.completed = (cont->timeRemaining <= loopDuration); + cont->timeRemaining -= loopDuration; if (loopDuration > 0) { - fastest = MIN(fastest, cont->intermediateResult.result.nanoSecPerRun); - cont->intermediateResult.result.nanoSecPerRun = fastest; - cont->state.nbLoops = (U32)(TIMELOOP_NANOSEC / fastest) + 1; + if(loopDuration >= MINUSABLETIME) { /* don't report results which have time too low */ + fastest = MIN(fastest, r.result.result.nanoSecPerRun); + } + r.result.result.nanoSecPerRun = fastest; + cont->fastestTime = fastest; + cont->nbLoops = (U32)(TIMELOOP_NANOSEC / fastest) + 1; } else { const unsigned multiplier = 2; - assert(cont->state.nbLoops < ((unsigned)-1) / multiplier); /* avoid overflow */ - cont->state.nbLoops *= multiplier; + assert(cont->nbLoops < ((unsigned)-1) / multiplier); /* avoid overflow */ + cont->nbLoops *= multiplier; } if(loopDuration < MINUSABLETIME) { /* don't report results which have time too low */ continue; @@ -397,6 +412,7 @@ void BMK_benchFunctionTimed( } completed = 1; } + return r; } /* benchMem with no allocation */ @@ -406,6 +422,7 @@ static BMK_return_t BMK_benchMemAdvancedNoAlloc( void** const resPtrs, size_t* const resSizes, void* resultBuffer, void* compressedBuffer, const size_t maxCompressedSize, + BMK_timedFnState_t* timeState, const void* srcBuffer, size_t srcSize, const size_t* fileSizes, unsigned nbFiles, @@ -504,21 +521,22 @@ static BMK_return_t BMK_benchMemAdvancedNoAlloc( /* Compression */ DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->\r", marks[markNb], displayName, (U32)srcSize); if(adv->loopMode == BMK_timeMode) { - BMK_customResultContinuation_t cont = BMK_init_customResultContinuation(adv->nbSeconds); - while(!cont.completed) { - BMK_benchFunctionTimed(&local_defaultCompress, (void*)ctx, &local_initCCtx, (void*)&cctxprep, - nbBlocks, srcPtrs, srcSizes, cPtrs, cSizes, &cont); - if(cont.intermediateResult.error) { - results.error = cont.intermediateResult.error; + BMK_customTimedReturn_t intermediateResult; + intermediateResult.completed = 0; + while(!intermediateResult.completed) { + intermediateResult = BMK_benchFunctionTimed(timeState, &local_defaultCompress, (void*)ctx, &local_initCCtx, (void*)&cctxprep, + nbBlocks, srcPtrs, srcSizes, cPtrs, cSizes); + if(intermediateResult.result.error) { + results.error = intermediateResult.result.error; return results; } - ratio = (double)(srcSize / cont.intermediateResult.result.sumOfReturn); + ratio = (double)(srcSize / intermediateResult.result.result.sumOfReturn); { int const ratioAccuracy = (ratio < 10.) ? 3 : 2; - double const compressionSpeed = ((double)srcSize / cont.intermediateResult.result.nanoSecPerRun) * 1000; + double const compressionSpeed = ((double)srcSize / intermediateResult.result.result.nanoSecPerRun) * 1000; int const cSpeedAccuracy = (compressionSpeed < 10.) ? 2 : 1; results.result.cSpeed = compressionSpeed * 1000000; - results.result.cSize = cont.intermediateResult.result.sumOfReturn; + results.result.cSize = intermediateResult.result.result.sumOfReturn; ratio = (double)srcSize / results.result.cSize; markNb = (markNb+1) % NB_MARKS; DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.*f),%6.*f MB/s\r", @@ -562,19 +580,21 @@ static BMK_return_t BMK_benchMemAdvancedNoAlloc( dctxprep.dictBuffer = dictBuffer; dctxprep.dictBufferSize = dictBufferSize; if(adv->loopMode == BMK_timeMode) { - BMK_customResultContinuation_t cont = BMK_init_customResultContinuation(adv->nbSeconds); - while(!cont.completed) { - BMK_benchFunctionTimed(&local_defaultDecompress, (void*)(dctx), &local_initDCtx, (void*)&dctxprep, - nbBlocks, (const void * const *)cPtrs, cSizes, resPtrs, resSizes, &cont); - if(cont.intermediateResult.error) { - results.error = cont.intermediateResult.error; + BMK_customTimedReturn_t intermediateResult; + intermediateResult.completed = 0; + BMK_resetTimeState(timeState, adv->nbSeconds); + while(!intermediateResult.completed) { + intermediateResult = BMK_benchFunctionTimed(timeState, &local_defaultDecompress, (void*)(dctx), &local_initDCtx, (void*)&dctxprep, + nbBlocks, (const void* const*)cPtrs, cSizes, resPtrs, resSizes); + if(intermediateResult.result.error) { + results.error = intermediateResult.result.error; return results; } { int const ratioAccuracy = (ratio < 10.) ? 3 : 2; double const compressionSpeed = results.result.cSpeed / 1000000; int const cSpeedAccuracy = (compressionSpeed < 10.) ? 2 : 1; - double const decompressionSpeed = ((double)srcSize / cont.intermediateResult.result.nanoSecPerRun) * 1000; + double const decompressionSpeed = ((double)srcSize / intermediateResult.result.result.nanoSecPerRun) * 1000; results.result.dSpeed = decompressionSpeed * 1000000; markNb = (markNb+1) % NB_MARKS; DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.*f),%6.*f MB/s ,%6.1f MB/s \r", @@ -588,7 +608,7 @@ static BMK_return_t BMK_benchMemAdvancedNoAlloc( decompressionResults = BMK_benchFunction( &local_defaultDecompress, (void*)(dctx), &local_initDCtx, (void*)&dctxprep, nbBlocks, - (const void * const *)cPtrs, cSizes, resPtrs, resSizes, + (const void* const*)cPtrs, cSizes, resPtrs, resSizes, adv->nbSeconds); if(decompressionResults.error) { results.error = decompressionResults.error; @@ -680,7 +700,7 @@ BMK_return_t BMK_benchMemAdvanced(const void* srcBuffer, size_t srcSize, U32 const maxNbBlocks = (U32) ((srcSize + (blockSize-1)) / blockSize) + nbFiles; /* these are the blockTable parameters, just split up */ - const void ** const srcPtrs = (const void ** const)malloc(maxNbBlocks * sizeof(void*)); + const void ** const srcPtrs = (const void** const)malloc(maxNbBlocks * sizeof(void*)); size_t* const srcSizes = (size_t* const)malloc(maxNbBlocks * sizeof(size_t)); void ** const cPtrs = (void** const)malloc(maxNbBlocks * sizeof(void*)); @@ -692,17 +712,20 @@ BMK_return_t BMK_benchMemAdvanced(const void* srcBuffer, size_t srcSize, const size_t maxCompressedSize = ZSTD_compressBound(srcSize) + (maxNbBlocks * 1024); /* add some room for safety */ void* compressedBuffer = malloc(maxCompressedSize); void* resultBuffer = malloc(srcSize); - + BMK_timedFnState_t* timeState = BMK_createTimeState(adv->nbSeconds); + + BMK_return_t results; int allocationincomplete = !compressedBuffer || !resultBuffer || !srcPtrs || !srcSizes || !cPtrs || !cSizes || !resPtrs || !resSizes; if (!allocationincomplete) { results = BMK_benchMemAdvancedNoAlloc(srcPtrs, srcSizes, cPtrs, cSizes, - resPtrs, resSizes, resultBuffer, compressedBuffer, maxCompressedSize, + resPtrs, resSizes, resultBuffer, compressedBuffer, maxCompressedSize, timeState, srcBuffer, srcSize, fileSizes, nbFiles, cLevel, comprParams, dictBuffer, dictBufferSize, ctx, dctx, displayLevel, displayName, adv); } /* clean up */ + BMK_freeTimeState(timeState); free(compressedBuffer); free(resultBuffer); @@ -741,7 +764,7 @@ static BMK_return_t BMK_benchMemCtxless(const void* srcBuffer, size_t srcSize, int cLevel, const ZSTD_compressionParameters* const comprParams, const void* dictBuffer, size_t dictBufferSize, int displayLevel, const char* displayName, - const BMK_advancedParams_t * const adv) + const BMK_advancedParams_t* const adv) { BMK_return_t res; ZSTD_CCtx* ctx = ZSTD_createCCtx(); @@ -854,11 +877,11 @@ BMK_return_t BMK_benchFilesAdvanced(const char* const * const fileNamesTable, un const ZSTD_compressionParameters* const compressionParams, int displayLevel, const BMK_advancedParams_t * const adv) { - void* srcBuffer; + void* srcBuffer = NULL; size_t benchedSize; void* dictBuffer = NULL; size_t dictBufferSize = 0; - size_t* fileSizes; + size_t* fileSizes = NULL; BMK_return_t res; U64 const totalSizeToLoad = UTIL_getTotalFileSize(fileNamesTable, nbFiles); @@ -890,7 +913,7 @@ BMK_return_t BMK_benchFilesAdvanced(const char* const * const fileNamesTable, un int errorCode = BMK_loadFiles(dictBuffer, dictBufferSize, fileSizes, &dictFileName, 1, displayLevel); if(errorCode) { res.error = errorCode; - return res; + goto _cleanUp; } } } @@ -912,7 +935,7 @@ BMK_return_t BMK_benchFilesAdvanced(const char* const * const fileNamesTable, un int errorCode = BMK_loadFiles(srcBuffer, benchedSize, fileSizes, fileNamesTable, nbFiles, displayLevel); if(errorCode) { res.error = errorCode; - return res; + goto _cleanUp; } } /* Bench */ @@ -929,7 +952,7 @@ BMK_return_t BMK_benchFilesAdvanced(const char* const * const fileNamesTable, un adv); } } - /* clean up */ +_cleanUp: free(srcBuffer); free(dictBuffer); free(fileSizes); @@ -966,7 +989,7 @@ BMK_return_t BMK_syntheticTest(int cLevel, double compressibility, displayLevel, name, adv); /* clean up */ - free((void*)srcBuffer); + free(srcBuffer); return res; } diff --git a/programs/bench.h b/programs/bench.h index 17cc0d0f5..87cf56380 100644 --- a/programs/bench.h +++ b/programs/bench.h @@ -26,8 +26,8 @@ extern "C" { * in .error != 0. */ #define ERROR_STRUCT(baseType, typeName) typedef struct { \ - int error; \ baseType result; \ + int error; \ } typeName typedef struct { @@ -36,44 +36,7 @@ typedef struct { double dSpeed; } BMK_result_t; -typedef struct { - size_t sumOfReturn; /* sum of return values */ - U64 nanoSecPerRun; /* time per iteration */ -} BMK_customResult_t; -//we might need a nbRuns or nbSecs if we're keeping timeMode / iterMode respectively. -//give benchMem responsibility to incrementally update display. - ERROR_STRUCT(BMK_result_t, BMK_return_t); -ERROR_STRUCT(BMK_customResult_t, BMK_customReturn_t); - -typedef enum { - BMK_timeMode = 0, - BMK_iterMode = 1 -} BMK_loopMode_t; - -typedef enum { - BMK_both = 0, - BMK_decodeOnly = 1, - BMK_compressOnly = 2 -} BMK_mode_t; - -typedef struct { - BMK_mode_t mode; /* 0: all, 1: compress only 2: decode only */ - BMK_loopMode_t loopMode; /* if loopmode, then nbSeconds = nbLoops */ - unsigned nbSeconds; /* default timing is in nbSeconds */ - size_t blockSize; /* Maximum allowable size of a block*/ - unsigned nbWorkers; /* multithreading */ - unsigned realTime; /* real time priority */ - int additionalParam; /* used by python speed benchmark */ - unsigned ldmFlag; /* enables long distance matching */ - unsigned ldmMinMatch; /* below: parameters for long distance matching, see zstd.1.md for meaning */ - unsigned ldmHashLog; - unsigned ldmBucketSizeLog; - unsigned ldmHashEveryLog; -} BMK_advancedParams_t; - -/* returns default parameters used by nonAdvanced functions */ -BMK_advancedParams_t BMK_initAdvancedParams(void); /* called in cli */ /* Loads files in fileNamesTable into memory, as well as a dictionary @@ -100,6 +63,35 @@ BMK_return_t BMK_benchFiles(const char* const * const fileNamesTable, unsigned c int const cLevel, const ZSTD_compressionParameters* const compressionParams, int displayLevel); +typedef enum { + BMK_timeMode = 0, + BMK_iterMode = 1 +} BMK_loopMode_t; + +typedef enum { + BMK_both = 0, + BMK_decodeOnly = 1, + BMK_compressOnly = 2 +} BMK_mode_t; + +typedef struct { + BMK_mode_t mode; /* 0: all, 1: compress only 2: decode only */ + BMK_loopMode_t loopMode; /* if loopmode, then nbSeconds = nbLoops */ + unsigned nbSeconds; /* default timing is in nbSeconds */ + size_t blockSize; /* Maximum allowable size of a block*/ + unsigned nbWorkers; /* multithreading */ + unsigned realTime; /* real time priority */ + int additionalParam; /* used by python speed benchmark */ + unsigned ldmFlag; /* enables long distance matching */ + unsigned ldmMinMatch; /* below: parameters for long distance matching, see zstd.1.md for meaning */ + unsigned ldmHashLog; + unsigned ldmBucketSizeLog; + unsigned ldmHashEveryLog; +} BMK_advancedParams_t; + +/* returns default parameters used by nonAdvanced functions */ +BMK_advancedParams_t BMK_initAdvancedParams(void); + /* See benchFiles for normal parameter uses and return, see advancedParams_t for adv */ BMK_return_t BMK_benchFilesAdvanced(const char* const * const fileNamesTable, unsigned const nbFiles, const char* const dictFileName, @@ -158,10 +150,20 @@ BMK_return_t BMK_benchMemAdvanced(const void* srcBuffer, size_t srcSize, int displayLevel, const char* displayName, const BMK_advancedParams_t* adv); +typedef struct { + size_t sumOfReturn; /* sum of return values */ + U64 nanoSecPerRun; /* time per iteration */ +} BMK_customResult_t; + +ERROR_STRUCT(BMK_customResult_t, BMK_customReturn_t); + +typedef size_t (*BMK_benchFn_t)(const void*, size_t, void*, size_t, void*); +typedef size_t (*BMK_initFn_t)(void*); + /* This function times the execution of 2 argument functions, benchFn and initFn */ /* benchFn - (*benchFn)(srcBuffers[i], srcSizes[i], dstBuffers[i], dstCapacities[i], benchPayload) - * is run iter times + * is run nbLoops times * initFn - (*initFn)(initPayload) is run once per benchmark at the beginning. This argument can * be NULL, in which case nothing is run. * blockCount - number of blocks (size of srcBuffers, srcSizes, dstBuffers, dstCapacities) @@ -169,7 +171,7 @@ BMK_return_t BMK_benchMemAdvanced(const void* srcBuffer, size_t srcSize, * srcSizes - an array of the sizes of above buffers * dstBuffers - an array of buffers to be written into by benchFn * dstCapacities - an array of the capacities of above buffers. - * iter - defines number of times benchFn is run. + * nbLoops - defines number of times benchFn is run. * return * .error will give a nonzero value if ZSTD_isError() is nonzero for any of the return * of the calls to initFn and benchFn, or if benchFunction errors internally @@ -181,44 +183,40 @@ BMK_return_t BMK_benchMemAdvanced(const void* srcBuffer, size_t srcSize, * dstBuffer. */ BMK_customReturn_t BMK_benchFunction( - size_t (*benchFn)(const void*, size_t, void*, size_t, void*), void* benchPayload, - size_t (*initFn)(void*), void* initPayload, + BMK_benchFn_t benchFn, void* benchPayload, + BMK_initFn_t initFn, void* initPayload, size_t blockCount, const void* const * const srcBuffers, const size_t* srcSizes, void* const * const dstBuffers, const size_t* dstCapacities, - unsigned sec); + unsigned nbLoops); -typedef struct { - unsigned nbLoops; - U64 timeRemaining; - UTIL_time_t coolTime; -} BMK_timeState_t; + +/* state information needed to advance computation for benchFunctionTimed */ +typedef struct BMK_timeState_t BMK_timedFnState_t; +/* initializes timeState object with desired number of seconds */ +BMK_timedFnState_t* BMK_createTimeState(unsigned nbSeconds); +/* resets existing timeState object */ +void BMK_resetTimeState(BMK_timedFnState_t*, unsigned nbSeconds); +/* deletes timeState object */ +void BMK_freeTimeState(BMK_timedFnState_t* state); typedef struct { + BMK_customReturn_t result; int completed; - BMK_customReturn_t intermediateResult; /* since the wrapper can't err, don't need ERROR_STRUCT(cRC, just check here) */ - BMK_timeState_t state; -} BMK_customResultContinuation_t; +} BMK_customTimedReturn_t; /* - * initializes the last argument of benchFunctionTimed, with iter being the number of seconds to bench (see below) - */ -BMK_customResultContinuation_t BMK_init_customResultContinuation(unsigned iter); - -/* - * Benchmarks custom functions like BMK_benchFunction(), but runs for iter seconds rather than a fixed number of iterations + * Benchmarks custom functions like BMK_benchFunction(), but runs for nbSeconds seconds rather than a fixed number of loops * arguments mostly the same other than BMK_benchFunction() - * Usage - benchFunctionTimed will return in approximately one second, where the intermediate results can be found in - * the *cont passed in and be displayed/used as wanted. Keep calling BMK_benchFunctionTimed() until cont->completed = 1 - * to continue updating intermediate result. + * Usage - benchFunctionTimed will return in approximately one second. Keep calling BMK_benchFunctionTimed() until the return's completed field = 1. + * to continue updating intermediate result. Intermediate return values are returned by the function. */ -void BMK_benchFunctionTimed( - size_t (*benchFn)(const void*, size_t, void*, size_t, void*), void* benchPayload, - size_t (*initFn)(void*), void* initPayload, +BMK_customTimedReturn_t BMK_benchFunctionTimed(BMK_timedFnState_t* cont, + BMK_benchFn_t benchFn, void* benchPayload, + BMK_initFn_t initFn, void* initPayload, size_t blockCount, const void* const * const srcBlockBuffers, const size_t* srcBlockSizes, - void* const * const dstBlockBuffers, const size_t* dstBlockCapacities, - BMK_customResultContinuation_t* cont); + void* const * const dstBlockBuffers, const size_t* dstBlockCapacities); #endif /* BENCH_H_121279284357 */ diff --git a/programs/zstdcli.c b/programs/zstdcli.c index a450ecac2..b3e0c0f63 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -710,13 +710,8 @@ int main(int argCount, const char* argv[]) /* Select compressibility of synthetic sample */ case 'P': - { U32 proba32 = 0; - while ((argument[1]>= '0') && (argument[1]<= '9')) { - proba32 *= 10; - proba32 += argument[1] - '0'; - argument++; - } - compressibility = (double)proba32 / 100; + { argument++; + compressibility = (double)readU32FromChar(&argument) / 100; } break; diff --git a/tests/fullbench.c b/tests/fullbench.c index 0a18eb2d9..b548a33f3 100644 --- a/tests/fullbench.c +++ b/tests/fullbench.c @@ -290,7 +290,7 @@ static size_t benchMem(const void* src, size_t srcSize, U32 benchNb) size_t const dstBuffSize = ZSTD_compressBound(srcSize); void* buff2; const char* benchName; - size_t (*benchFunction)(const void* src, size_t srcSize, void* dst, size_t dstSize, void* verifBuff); + BMK_benchFn_t benchFunction; BMK_customReturn_t r; int errorcode = 0; From 50d612f4f0005c3ba793eb4f374e9f34a8c4601b Mon Sep 17 00:00:00 2001 From: George Lu Date: Mon, 25 Jun 2018 15:01:03 -0700 Subject: [PATCH 010/372] Interleave compression/decompression Fix Bugs --- programs/bench.c | 219 ++++++++++++++++++++++++----------------------- 1 file changed, 113 insertions(+), 106 deletions(-) diff --git a/programs/bench.c b/programs/bench.c index b2b5dc3b2..a54168c42 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -394,12 +394,12 @@ BMK_customTimedReturn_t BMK_benchFunctionTimed( r.completed = (cont->timeRemaining <= loopDuration); cont->timeRemaining -= loopDuration; if (loopDuration > 0) { - if(loopDuration >= MINUSABLETIME) { /* don't report results which have time too low */ - fastest = MIN(fastest, r.result.result.nanoSecPerRun); + fastest = MIN(fastest, r.result.result.nanoSecPerRun); + if(loopDuration >= MINUSABLETIME) { + r.result.result.nanoSecPerRun = fastest; + cont->fastestTime = fastest; } - r.result.result.nanoSecPerRun = fastest; - cont->fastestTime = fastest; - cont->nbLoops = (U32)(TIMELOOP_NANOSEC / fastest) + 1; + cont->nbLoops = (U32)(TIMELOOP_NANOSEC / r.result.result.nanoSecPerRun) + 1; } else { const unsigned multiplier = 2; assert(cont->nbLoops < ((unsigned)-1) / multiplier); /* avoid overflow */ @@ -422,7 +422,7 @@ static BMK_return_t BMK_benchMemAdvancedNoAlloc( void** const resPtrs, size_t* const resSizes, void* resultBuffer, void* compressedBuffer, const size_t maxCompressedSize, - BMK_timedFnState_t* timeState, + BMK_timedFnState_t* timeStateCompress, BMK_timedFnState_t* timeStateDecompress, const void* srcBuffer, size_t srcSize, const size_t* fileSizes, unsigned nbFiles, @@ -509,34 +509,98 @@ static BMK_return_t BMK_benchMemAdvancedNoAlloc( U32 markNb = 0; DISPLAYLEVEL(2, "\r%79s\r", ""); - - if (adv->mode != BMK_decodeOnly) { + DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->\r", marks[markNb], displayName, (U32)srcSize); + { BMK_initCCtxArgs cctxprep; + BMK_initDCtxArgs dctxprep; cctxprep.ctx = ctx; cctxprep.dictBuffer = dictBuffer; cctxprep.dictBufferSize = dictBufferSize; cctxprep.cLevel = cLevel; cctxprep.comprParams = comprParams; cctxprep.adv = adv; - /* Compression */ - DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->\r", marks[markNb], displayName, (U32)srcSize); + dctxprep.dctx = dctx; + dctxprep.dictBuffer = dictBuffer; + dctxprep.dictBufferSize = dictBufferSize; if(adv->loopMode == BMK_timeMode) { - BMK_customTimedReturn_t intermediateResult; - intermediateResult.completed = 0; - while(!intermediateResult.completed) { - intermediateResult = BMK_benchFunctionTimed(timeState, &local_defaultCompress, (void*)ctx, &local_initCCtx, (void*)&cctxprep, - nbBlocks, srcPtrs, srcSizes, cPtrs, cSizes); - if(intermediateResult.result.error) { - results.error = intermediateResult.result.error; + BMK_customTimedReturn_t intermediateResultCompress; + BMK_customTimedReturn_t intermediateResultDecompress; + if(adv->mode == BMK_compressOnly) { + intermediateResultCompress.completed = 0; + intermediateResultDecompress.completed = 1; + } else if (adv->mode == BMK_decodeOnly) { + intermediateResultCompress.completed = 1; + intermediateResultDecompress.completed = 0; + } else { /* both */ + intermediateResultCompress.completed = 0; + intermediateResultDecompress.completed = 0; + } + while(!(intermediateResultCompress.completed && intermediateResultDecompress.completed)) { + if(!intermediateResultCompress.completed) { + intermediateResultCompress = BMK_benchFunctionTimed(timeStateCompress, &local_defaultCompress, (void*)ctx, &local_initCCtx, (void*)&cctxprep, + nbBlocks, srcPtrs, srcSizes, cPtrs, cSizes); + if(intermediateResultCompress.result.error) { + results.error = intermediateResultCompress.result.error; + return results; + } + ratio = (double)(srcSize / intermediateResultCompress.result.result.sumOfReturn); + { + int const ratioAccuracy = (ratio < 10.) ? 3 : 2; + double const compressionSpeed = ((double)srcSize / intermediateResultCompress.result.result.nanoSecPerRun) * 1000; + int const cSpeedAccuracy = (compressionSpeed < 10.) ? 2 : 1; + results.result.cSpeed = compressionSpeed * 1000000; + results.result.cSize = intermediateResultCompress.result.result.sumOfReturn; + ratio = (double)srcSize / results.result.cSize; + markNb = (markNb+1) % NB_MARKS; + DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.*f),%6.*f MB/s\r", + marks[markNb], displayName, (U32)srcSize, (U32)results.result.cSize, + ratioAccuracy, ratio, + cSpeedAccuracy, compressionSpeed); + } + } + + if(!intermediateResultDecompress.completed) { + intermediateResultDecompress = BMK_benchFunctionTimed(timeStateDecompress, &local_defaultDecompress, (void*)(dctx), &local_initDCtx, (void*)&dctxprep, + nbBlocks, (const void* const*)cPtrs, cSizes, resPtrs, resSizes); + if(intermediateResultDecompress.result.error) { + results.error = intermediateResultDecompress.result.error; + return results; + } + + { + int const ratioAccuracy = (ratio < 10.) ? 3 : 2; + double const compressionSpeed = results.result.cSpeed / 1000000; + int const cSpeedAccuracy = (compressionSpeed < 10.) ? 2 : 1; + double const decompressionSpeed = ((double)srcSize / intermediateResultDecompress.result.result.nanoSecPerRun) * 1000; + results.result.dSpeed = decompressionSpeed * 1000000; + markNb = (markNb+1) % NB_MARKS; + DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.*f),%6.*f MB/s ,%6.1f MB/s \r", + marks[markNb], displayName, (U32)srcSize, (U32)results.result.cSize, + ratioAccuracy, ratio, + cSpeedAccuracy, compressionSpeed, + decompressionSpeed); + } + } + } + + } else { + if(adv->mode != BMK_decodeOnly) { + BMK_customReturn_t compressionResults = BMK_benchFunction(&local_defaultCompress, (void*)ctx, &local_initCCtx, (void*)&cctxprep, + nbBlocks, srcPtrs, srcSizes, cPtrs, cSizes, adv->nbSeconds); + if(compressionResults.error) { + results.error = compressionResults.error; return results; } - ratio = (double)(srcSize / intermediateResult.result.result.sumOfReturn); + if(compressionResults.result.nanoSecPerRun == 0) { + results.result.cSpeed = 0; + } else { + results.result.cSpeed = (double)srcSize / compressionResults.result.nanoSecPerRun * TIMELOOP_NANOSEC; + } + results.result.cSize = compressionResults.result.sumOfReturn; { int const ratioAccuracy = (ratio < 10.) ? 3 : 2; - double const compressionSpeed = ((double)srcSize / intermediateResult.result.result.nanoSecPerRun) * 1000; + double const compressionSpeed = results.result.cSpeed / 1000000; int const cSpeedAccuracy = (compressionSpeed < 10.) ? 2 : 1; - results.result.cSpeed = compressionSpeed * 1000000; - results.result.cSize = intermediateResult.result.result.sumOfReturn; ratio = (double)srcSize / results.result.cSize; markNb = (markNb+1) % NB_MARKS; DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.*f),%6.*f MB/s\r", @@ -545,91 +609,33 @@ static BMK_return_t BMK_benchMemAdvancedNoAlloc( cSpeedAccuracy, compressionSpeed); } } - } else { - BMK_customReturn_t compressionResults = BMK_benchFunction(&local_defaultCompress, (void*)ctx, &local_initCCtx, (void*)&cctxprep, - nbBlocks, srcPtrs, srcSizes, cPtrs, cSizes, adv->nbSeconds); - if(compressionResults.error) { - results.error = compressionResults.error; - return results; - } - if(compressionResults.result.nanoSecPerRun == 0) { - results.result.cSpeed = 0; - } else { - results.result.cSpeed = (double)srcSize / compressionResults.result.nanoSecPerRun * TIMELOOP_NANOSEC; - } - results.result.cSize = compressionResults.result.sumOfReturn; - { - int const ratioAccuracy = (ratio < 10.) ? 3 : 2; - double const compressionSpeed = results.result.cSpeed / 1000000; - int const cSpeedAccuracy = (compressionSpeed < 10.) ? 2 : 1; - ratio = (double)srcSize / results.result.cSize; - markNb = (markNb+1) % NB_MARKS; - DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.*f),%6.*f MB/s\r", - marks[markNb], displayName, (U32)srcSize, (U32)results.result.cSize, - ratioAccuracy, ratio, - cSpeedAccuracy, compressionSpeed); - } - } - - } /* if (adv->mode != BMK_decodeOnly) */ - - if(adv->mode != BMK_compressOnly) { - BMK_initDCtxArgs dctxprep; - BMK_customReturn_t decompressionResults; - dctxprep.dctx = dctx; - dctxprep.dictBuffer = dictBuffer; - dctxprep.dictBufferSize = dictBufferSize; - if(adv->loopMode == BMK_timeMode) { - BMK_customTimedReturn_t intermediateResult; - intermediateResult.completed = 0; - BMK_resetTimeState(timeState, adv->nbSeconds); - while(!intermediateResult.completed) { - intermediateResult = BMK_benchFunctionTimed(timeState, &local_defaultDecompress, (void*)(dctx), &local_initDCtx, (void*)&dctxprep, - nbBlocks, (const void* const*)cPtrs, cSizes, resPtrs, resSizes); - if(intermediateResult.result.error) { - results.error = intermediateResult.result.error; + if(adv->mode != BMK_compressOnly) { + BMK_customReturn_t decompressionResults = BMK_benchFunction( + &local_defaultDecompress, (void*)(dctx), + &local_initDCtx, (void*)&dctxprep, nbBlocks, + (const void* const*)cPtrs, cSizes, resPtrs, resSizes, + adv->nbSeconds); + if(decompressionResults.error) { + results.error = decompressionResults.error; return results; } - + if(decompressionResults.result.nanoSecPerRun == 0) { + results.result.dSpeed = 0; + } else { + results.result.dSpeed = (double)srcSize / decompressionResults.result.nanoSecPerRun * TIMELOOP_NANOSEC; + } { int const ratioAccuracy = (ratio < 10.) ? 3 : 2; - double const compressionSpeed = results.result.cSpeed / 1000000; - int const cSpeedAccuracy = (compressionSpeed < 10.) ? 2 : 1; - double const decompressionSpeed = ((double)srcSize / intermediateResult.result.result.nanoSecPerRun) * 1000; - results.result.dSpeed = decompressionSpeed * 1000000; - markNb = (markNb+1) % NB_MARKS; - DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.*f),%6.*f MB/s ,%6.1f MB/s \r", - marks[markNb], displayName, (U32)srcSize, (U32)results.result.cSize, - ratioAccuracy, ratio, - cSpeedAccuracy, compressionSpeed, - decompressionSpeed); + double const compressionSpeed = results.result.cSpeed / 1000000; + int const cSpeedAccuracy = (compressionSpeed < 10.) ? 2 : 1; + double const decompressionSpeed = ((double)srcSize / decompressionResults.result.nanoSecPerRun) * 1000; + results.result.dSpeed = decompressionSpeed * 1000000; + markNb = (markNb+1) % NB_MARKS; + DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.*f),%6.*f MB/s ,%6.1f MB/s \r", + marks[markNb], displayName, (U32)srcSize, (U32)results.result.cSize, + ratioAccuracy, ratio, + cSpeedAccuracy, compressionSpeed, + decompressionSpeed); } - } - } else { - decompressionResults = BMK_benchFunction( - &local_defaultDecompress, (void*)(dctx), - &local_initDCtx, (void*)&dctxprep, nbBlocks, - (const void* const*)cPtrs, cSizes, resPtrs, resSizes, - adv->nbSeconds); - if(decompressionResults.error) { - results.error = decompressionResults.error; - return results; - } - if(decompressionResults.result.nanoSecPerRun == 0) { - results.result.dSpeed = 0; - } else { - results.result.dSpeed = (double)srcSize / decompressionResults.result.nanoSecPerRun * TIMELOOP_NANOSEC; - } - { int const ratioAccuracy = (ratio < 10.) ? 3 : 2; - double const compressionSpeed = results.result.cSpeed / 1000000; - int const cSpeedAccuracy = (compressionSpeed < 10.) ? 2 : 1; - double const decompressionSpeed = ((double)srcSize / decompressionResults.result.nanoSecPerRun) * 1000; - results.result.dSpeed = decompressionSpeed * 1000000; - markNb = (markNb+1) % NB_MARKS; - DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.*f),%6.*f MB/s ,%6.1f MB/s \r", - marks[markNb], displayName, (U32)srcSize, (U32)results.result.cSize, - ratioAccuracy, ratio, - cSpeedAccuracy, compressionSpeed, - decompressionSpeed); } } } @@ -712,20 +718,21 @@ BMK_return_t BMK_benchMemAdvanced(const void* srcBuffer, size_t srcSize, const size_t maxCompressedSize = ZSTD_compressBound(srcSize) + (maxNbBlocks * 1024); /* add some room for safety */ void* compressedBuffer = malloc(maxCompressedSize); void* resultBuffer = malloc(srcSize); - BMK_timedFnState_t* timeState = BMK_createTimeState(adv->nbSeconds); - + BMK_timedFnState_t* timeStateCompress = BMK_createTimeState(adv->nbSeconds); + BMK_timedFnState_t* timeStateDecompress = BMK_createTimeState(adv->nbSeconds); BMK_return_t results; int allocationincomplete = !compressedBuffer || !resultBuffer || !srcPtrs || !srcSizes || !cPtrs || !cSizes || !resPtrs || !resSizes; if (!allocationincomplete) { results = BMK_benchMemAdvancedNoAlloc(srcPtrs, srcSizes, cPtrs, cSizes, - resPtrs, resSizes, resultBuffer, compressedBuffer, maxCompressedSize, timeState, + resPtrs, resSizes, resultBuffer, compressedBuffer, maxCompressedSize, timeStateCompress, timeStateDecompress, srcBuffer, srcSize, fileSizes, nbFiles, cLevel, comprParams, dictBuffer, dictBufferSize, ctx, dctx, displayLevel, displayName, adv); } /* clean up */ - BMK_freeTimeState(timeState); + BMK_freeTimeState(timeStateCompress); + BMK_freeTimeState(timeStateDecompress); free(compressedBuffer); free(resultBuffer); From ceb4b9e6701a6d778116c0c0de824f3d10dcf9d9 Mon Sep 17 00:00:00 2001 From: George Lu Date: Tue, 19 Jun 2018 17:07:34 -0700 Subject: [PATCH 011/372] New fullbench args -l# gives some level --zstd= style parameters also supported --- tests/fullbench.c | 217 ++++++++++++++++++++++++++++++++++------------ 1 file changed, 163 insertions(+), 54 deletions(-) diff --git a/tests/fullbench.c b/tests/fullbench.c index b548a33f3..7859745a3 100644 --- a/tests/fullbench.c +++ b/tests/fullbench.c @@ -46,6 +46,8 @@ #define KNUTH 2654435761U #define MAX_MEM (1984 MB) +#define DEFAULT_CLEVEL 1 + #define COMPRESSIBILITY_DEFAULT 0.50 static const size_t g_sampleSize = 10000000; @@ -90,15 +92,59 @@ static size_t BMK_findMaxMem(U64 requiredMem) return (size_t) requiredMem; } +/*_******************************************************* +* Argument Parsing +*********************************************************/ + +#define ERROR_OUT(msg) { DISPLAY("%s \n", msg); exit(1); } + + static unsigned readU32FromChar(const char** stringPtr) +{ + const char errorMsg[] = "error: numeric value too large"; + unsigned result = 0; + while ((**stringPtr >='0') && (**stringPtr <='9')) { + unsigned const max = (((unsigned)(-1)) / 10) - 1; + if (result > max) ERROR_OUT(errorMsg); + result *= 10, result += **stringPtr - '0', (*stringPtr)++ ; + } + if ((**stringPtr=='K') || (**stringPtr=='M')) { + unsigned const maxK = ((unsigned)(-1)) >> 10; + if (result > maxK) ERROR_OUT(errorMsg); + result <<= 10; + if (**stringPtr=='M') { + if (result > maxK) ERROR_OUT(errorMsg); + result <<= 10; + } + (*stringPtr)++; /* skip `K` or `M` */ + if (**stringPtr=='i') (*stringPtr)++; + if (**stringPtr=='B') (*stringPtr)++; + } + return result; +} + +static unsigned longCommandWArg(const char** stringPtr, const char* longCommand) +{ + size_t const comSize = strlen(longCommand); + int const result = !strncmp(*stringPtr, longCommand, comSize); + if (result) *stringPtr += comSize; + return result; +} /*_******************************************************* * Benchmark wrappers *********************************************************/ + +static ZSTD_CCtx* g_zcc = NULL; + size_t local_ZSTD_compress(const void* src, size_t srcSize, void* dst, size_t dstSize, void* buff2) { - (void)buff2; - return ZSTD_compress(dst, dstSize, src, srcSize, 1); + ZSTD_parameters p; + ZSTD_frameParameters f = {1 /* contentSizeHeader*/, 0, 0}; + p.fParams = f; + p.cParams = *(ZSTD_compressionParameters*)buff2; + return ZSTD_compress_advanced (g_zcc,dst, dstSize, src, srcSize, NULL ,0, p); + //return ZSTD_compress(dst, dstSize, src, srcSize, cLevel); } static size_t g_cSize = 0; @@ -132,8 +178,11 @@ size_t local_ZSTD_compressStream(const void* src, size_t srcSize, void* dst, siz { ZSTD_outBuffer buffOut; ZSTD_inBuffer buffIn; - (void)buff2; - ZSTD_initCStream(g_cstream, 1); + ZSTD_parameters p; + ZSTD_frameParameters f = {1 /* contentSizeHeader*/, 0, 0}; + p.fParams = f; + p.cParams = *(ZSTD_compressionParameters*)buff2; + ZSTD_initCStream_advanced(g_cstream, NULL, 0, p, ZSTD_CONTENTSIZE_UNKNOWN); buffOut.dst = dst; buffOut.size = dstCapacity; buffOut.pos = 0; @@ -150,7 +199,6 @@ static size_t local_ZSTD_compress_generic_end(const void* src, size_t srcSize, v ZSTD_outBuffer buffOut; ZSTD_inBuffer buffIn; (void)buff2; - ZSTD_CCtx_setParameter(g_cstream, ZSTD_p_compressionLevel, 1); buffOut.dst = dst; buffOut.size = dstCapacity; buffOut.pos = 0; @@ -166,7 +214,6 @@ static size_t local_ZSTD_compress_generic_continue(const void* src, size_t srcSi ZSTD_outBuffer buffOut; ZSTD_inBuffer buffIn; (void)buff2; - ZSTD_CCtx_setParameter(g_cstream, ZSTD_p_compressionLevel, 1); buffOut.dst = dst; buffOut.size = dstCapacity; buffOut.pos = 0; @@ -183,7 +230,6 @@ static size_t local_ZSTD_compress_generic_T2_end(const void* src, size_t srcSize ZSTD_outBuffer buffOut; ZSTD_inBuffer buffIn; (void)buff2; - ZSTD_CCtx_setParameter(g_cstream, ZSTD_p_compressionLevel, 1); ZSTD_CCtx_setParameter(g_cstream, ZSTD_p_nbWorkers, 2); buffOut.dst = dst; buffOut.size = dstCapacity; @@ -200,7 +246,6 @@ static size_t local_ZSTD_compress_generic_T2_continue(const void* src, size_t sr ZSTD_outBuffer buffOut; ZSTD_inBuffer buffIn; (void)buff2; - ZSTD_CCtx_setParameter(g_cstream, ZSTD_p_compressionLevel, 1); ZSTD_CCtx_setParameter(g_cstream, ZSTD_p_nbWorkers, 2); buffOut.dst = dst; buffOut.size = dstCapacity; @@ -230,13 +275,14 @@ static size_t local_ZSTD_decompressStream(const void* src, size_t srcSize, void* return buffOut.pos; } -static ZSTD_CCtx* g_zcc = NULL; - #ifndef ZSTD_DLL_IMPORT size_t local_ZSTD_compressContinue(const void* src, size_t srcSize, void* dst, size_t dstCapacity, void* buff2) { - (void)buff2; - ZSTD_compressBegin(g_zcc, 1 /* compressionLevel */); + ZSTD_parameters p; + ZSTD_frameParameters f = {1 /* contentSizeHeader*/, 0, 0}; + p.fParams = f; + p.cParams = *(ZSTD_compressionParameters*)buff2; + ZSTD_compressBegin_advanced(g_zcc, NULL, 0, p, srcSize); return ZSTD_compressEnd(g_zcc, dst, dstCapacity, src, srcSize); } @@ -244,10 +290,13 @@ size_t local_ZSTD_compressContinue(const void* src, size_t srcSize, void* dst, s size_t local_ZSTD_compressContinue_extDict(const void* src, size_t srcSize, void* dst, size_t dstCapacity, void* buff2) { BYTE firstBlockBuf[FIRST_BLOCK_SIZE]; - - (void)buff2; + + ZSTD_parameters p; + ZSTD_frameParameters f = {1 , 0, 0}; + p.fParams = f; + p.cParams = *(ZSTD_compressionParameters*)buff2; + ZSTD_compressBegin_advanced(g_zcc, NULL, 0, p, srcSize); memcpy(firstBlockBuf, src, FIRST_BLOCK_SIZE); - ZSTD_compressBegin(g_zcc, 1); { size_t const compressResult = ZSTD_compressContinue(g_zcc, dst, dstCapacity, firstBlockBuf, FIRST_BLOCK_SIZE); if (ZSTD_isError(compressResult)) { DISPLAY("local_ZSTD_compressContinue_extDict error : %s\n", ZSTD_getErrorName(compressResult)); return compressResult; } @@ -284,11 +333,11 @@ size_t local_ZSTD_decompressContinue(const void* src, size_t srcSize, void* dst, /*_******************************************************* * Bench functions *********************************************************/ -static size_t benchMem(const void* src, size_t srcSize, U32 benchNb) +static size_t benchMem(const void* src, size_t srcSize, U32 benchNb, int cLevel, ZSTD_compressionParameters* cparams) { BYTE* dstBuff; size_t const dstBuffSize = ZSTD_compressBound(srcSize); - void* buff2; + void* buff2, *buff1; const char* benchName; BMK_benchFn_t benchFunction; BMK_customReturn_t r; @@ -298,14 +347,14 @@ static size_t benchMem(const void* src, size_t srcSize, U32 benchNb) switch(benchNb) { case 1: - benchFunction = local_ZSTD_compress; benchName = "compress(1)"; + benchFunction = local_ZSTD_compress; benchName = "compress"; break; case 2: benchFunction = local_ZSTD_decompress; benchName = "decompress"; break; #ifndef ZSTD_DLL_IMPORT case 11: - benchFunction = local_ZSTD_compressContinue; benchName = "compressContinue(1)"; + benchFunction = local_ZSTD_compressContinue; benchName = "compressContinue"; break; case 12: benchFunction = local_ZSTD_compressContinue_extDict; benchName = "compressContinue_extDict"; @@ -321,7 +370,7 @@ static size_t benchMem(const void* src, size_t srcSize, U32 benchNb) break; #endif case 41: - benchFunction = local_ZSTD_compressStream; benchName = "compressStream(1)"; + benchFunction = local_ZSTD_compressStream; benchName = "compressStream"; break; case 42: benchFunction = local_ZSTD_decompressStream; benchName = "decompressStream"; @@ -350,26 +399,59 @@ static size_t benchMem(const void* src, size_t srcSize, U32 benchNb) free(dstBuff); free(buff2); return 12; } + buff1 = buff2; if (g_zcc==NULL) g_zcc = ZSTD_createCCtx(); if (g_zdc==NULL) g_zdc = ZSTD_createDCtx(); if (g_cstream==NULL) g_cstream = ZSTD_createCStream(); if (g_dstream==NULL) g_dstream = ZSTD_createDStream(); + /* DISPLAY("params: cLevel %d, wlog %d hlog %d clog %d slog %d slen %d tlen %d strat %d \n" + , cLevel, cparams->windowLog, cparams->hashLog, cparams->chainLog, cparams->searchLog, + cparams->searchLength, cparams->targetLength, cparams->strategy);*/ + + ZSTD_CCtx_setParameter(g_zcc, ZSTD_p_compressionLevel, cLevel); + ZSTD_CCtx_setParameter(g_zcc, ZSTD_p_windowLog, cparams->windowLog); + ZSTD_CCtx_setParameter(g_zcc, ZSTD_p_hashLog, cparams->hashLog); + ZSTD_CCtx_setParameter(g_zcc, ZSTD_p_chainLog, cparams->chainLog); + ZSTD_CCtx_setParameter(g_zcc, ZSTD_p_searchLog, cparams->searchLog); + ZSTD_CCtx_setParameter(g_zcc, ZSTD_p_minMatch, cparams->searchLength); + ZSTD_CCtx_setParameter(g_zcc, ZSTD_p_targetLength, cparams->targetLength); + ZSTD_CCtx_setParameter(g_zcc, ZSTD_p_compressionStrategy, cparams->strategy); + + + ZSTD_CCtx_setParameter(g_cstream, ZSTD_p_compressionLevel, cLevel); + ZSTD_CCtx_setParameter(g_cstream, ZSTD_p_windowLog, cparams->windowLog); + ZSTD_CCtx_setParameter(g_cstream, ZSTD_p_hashLog, cparams->hashLog); + ZSTD_CCtx_setParameter(g_cstream, ZSTD_p_chainLog, cparams->chainLog); + ZSTD_CCtx_setParameter(g_cstream, ZSTD_p_searchLog, cparams->searchLog); + ZSTD_CCtx_setParameter(g_cstream, ZSTD_p_minMatch, cparams->searchLength); + ZSTD_CCtx_setParameter(g_cstream, ZSTD_p_targetLength, cparams->targetLength); + ZSTD_CCtx_setParameter(g_cstream, ZSTD_p_compressionStrategy, cparams->strategy); + /* Preparation */ switch(benchNb) { + case 1: + buff2 = (void*)cparams; + break; case 2: - g_cSize = ZSTD_compress(buff2, dstBuffSize, src, srcSize, 1); + g_cSize = ZSTD_compress(buff2, dstBuffSize, src, srcSize, cLevel); break; #ifndef ZSTD_DLL_IMPORT + case 11: + buff2 = (void*)cparams; + break; + case 12: + buff2 = (void*)cparams; + break; case 13 : - g_cSize = ZSTD_compress(buff2, dstBuffSize, src, srcSize, 1); + g_cSize = ZSTD_compress(buff2, dstBuffSize, src, srcSize, cLevel); break; case 31: /* ZSTD_decodeLiteralsBlock */ { blockProperties_t bp; ZSTD_frameHeader zfp; size_t frameHeaderSize, skippedSize; - g_cSize = ZSTD_compress(dstBuff, dstBuffSize, src, srcSize, 1); + g_cSize = ZSTD_compress(dstBuff, dstBuffSize, src, srcSize, cLevel); frameHeaderSize = ZSTD_getFrameHeader(&zfp, dstBuff, ZSTD_frameHeaderSize_min); if (frameHeaderSize==0) frameHeaderSize = ZSTD_frameHeaderSize_min; ZSTD_getcBlockSize(dstBuff+frameHeaderSize, dstBuffSize, &bp); /* Get 1st block type */ @@ -389,8 +471,8 @@ static size_t benchMem(const void* src, size_t srcSize, U32 benchNb) const BYTE* ip = dstBuff; const BYTE* iend; size_t frameHeaderSize, cBlockSize; - ZSTD_compress(dstBuff, dstBuffSize, src, srcSize, 1); /* it would be better to use direct block compression here */ - g_cSize = ZSTD_compress(dstBuff, dstBuffSize, src, srcSize, 1); + ZSTD_compress(dstBuff, dstBuffSize, src, srcSize, cLevel); /* it would be better to use direct block compression here */ + g_cSize = ZSTD_compress(dstBuff, dstBuffSize, src, srcSize, cLevel); frameHeaderSize = ZSTD_getFrameHeader(&zfp, dstBuff, ZSTD_frameHeaderSize_min); if (frameHeaderSize==0) frameHeaderSize = ZSTD_frameHeaderSize_min; ip += frameHeaderSize; /* Skip frame Header */ @@ -412,8 +494,11 @@ static size_t benchMem(const void* src, size_t srcSize, U32 benchNb) case 31: goto _cleanOut; #endif + case 41 : + buff2 = (void*)cparams; + break; case 42 : - g_cSize = ZSTD_compress(buff2, dstBuffSize, src, srcSize, 1); + g_cSize = ZSTD_compress(buff2, dstBuffSize, src, srcSize, cLevel); break; /* test functions */ @@ -442,8 +527,8 @@ static size_t benchMem(const void* src, size_t srcSize, U32 benchNb) } _cleanOut: + free(buff1); free(dstBuff); - free(buff2); ZSTD_freeCCtx(g_zcc); g_zcc=NULL; ZSTD_freeDCtx(g_zdc); g_zdc=NULL; ZSTD_freeCStream(g_cstream); g_cstream=NULL; @@ -452,7 +537,7 @@ _cleanOut: } -static int benchSample(U32 benchNb) +static int benchSample(U32 benchNb, int cLevel, ZSTD_compressionParameters* cparams) { size_t const benchedSize = g_sampleSize; const char* name = "Sample 10MiB"; @@ -468,16 +553,16 @@ static int benchSample(U32 benchNb) DISPLAY("\r%79s\r", ""); DISPLAY(" %s : \n", name); if (benchNb) - benchMem(origBuff, benchedSize, benchNb); + benchMem(origBuff, benchedSize, benchNb, cLevel, cparams); else - for (benchNb=0; benchNb<100; benchNb++) benchMem(origBuff, benchedSize, benchNb); + for (benchNb=0; benchNb<100; benchNb++) benchMem(origBuff, benchedSize, benchNb, cLevel, cparams); free(origBuff); return 0; } -static int benchFiles(const char** fileNamesTable, const int nbFiles, U32 benchNb) +static int benchFiles(const char** fileNamesTable, const int nbFiles, U32 benchNb, int cLevel, ZSTD_compressionParameters* cparams) { /* Loop for each file */ int fileIdx; @@ -522,9 +607,9 @@ static int benchFiles(const char** fileNamesTable, const int nbFiles, U32 benchN DISPLAY("\r%79s\r", ""); DISPLAY(" %s : \n", inFileName); if (benchNb) - benchMem(origBuff, benchedSize, benchNb); + benchMem(origBuff, benchedSize, benchNb, cLevel, cparams); else - for (benchNb=0; benchNb<100; benchNb++) benchMem(origBuff, benchedSize, benchNb); + for (benchNb=0; benchNb<100; benchNb++) benchMem(origBuff, benchedSize, benchNb, cLevel, cparams); free(origBuff); } @@ -549,6 +634,8 @@ static int usage_advanced(const char* exename) DISPLAY( " -b# : test only function # \n"); DISPLAY( " -i# : iteration loops [1-9](default : %i)\n", NBLOOPS); DISPLAY( " -P# : sample compressibility (default : %.1f%%)\n", COMPRESSIBILITY_DEFAULT * 100); + DISPLAY( " -l# : benchmark functions at that compression level (default : %i)\n", DEFAULT_CLEVEL); + DISPLAY( " --zstd : custom parameter selection. Format same as zstdcli \n"); return 0; } @@ -565,6 +652,8 @@ int main(int argc, const char** argv) const char* exename = argv[0]; const char* input_filename = NULL; U32 benchNb = 0, main_pause = 0; + int cLevel = DEFAULT_CLEVEL; + ZSTD_compressionParameters cparams = ZSTD_getCParams(cLevel, 0, 0); DISPLAY(WELCOME_MESSAGE); if (argc<1) return badusage(exename); @@ -573,11 +662,29 @@ int main(int argc, const char** argv) const char* argument = argv[i]; assert(argument != NULL); - /* Commands (note : aggregated commands are allowed) */ - if (argument[0]=='-') { + if (longCommandWArg(&argument, "--zstd=")) { + for ( ; ;) { + if (longCommandWArg(&argument, "windowLog=") || longCommandWArg(&argument, "wlog=")) { cparams.windowLog = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; } + if (longCommandWArg(&argument, "chainLog=") || longCommandWArg(&argument, "clog=")) { cparams.chainLog = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; } + if (longCommandWArg(&argument, "hashLog=") || longCommandWArg(&argument, "hlog=")) { cparams.hashLog = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; } + if (longCommandWArg(&argument, "searchLog=") || longCommandWArg(&argument, "slog=")) { cparams.searchLog = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; } + if (longCommandWArg(&argument, "searchLength=") || longCommandWArg(&argument, "slen=")) { cparams.searchLength = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; } + if (longCommandWArg(&argument, "targetLength=") || longCommandWArg(&argument, "tlen=")) { cparams.targetLength = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; } + if (longCommandWArg(&argument, "strategy=") || longCommandWArg(&argument, "strat=")) { cparams.strategy = (ZSTD_strategy)(readU32FromChar(&argument)); if (argument[0]==',') { argument++; continue; } else break; } + if (longCommandWArg(&argument, "level=") || longCommandWArg(&argument, "lvl=")) { cLevel = (int)readU32FromChar(&argument); cparams = ZSTD_getCParams(cLevel, 0, 0); if (argument[0]==',') { argument++; continue; } else break; } + DISPLAY("invalid compression parameter \n"); + return 1; + } - while (argument[1]!=0) { - argument++; + if (argument[0] != 0) { + DISPLAY("invalid --zstd= format\n"); + return 1; // check the end of string + } else { + continue; + } + } else if (argument[0]=='-') { /* Commands (note : aggregated commands are allowed) */ + argument++; + while (argument[0]!=0) { switch(argument[0]) { @@ -590,34 +697,34 @@ int main(int argc, const char** argv) /* Select specific algorithm to bench */ case 'b': - benchNb = 0; - while ((argument[1]>= '0') && (argument[1]<= '9')) { - benchNb *= 10; - benchNb += argument[1] - '0'; + { argument++; + benchNb = readU32FromChar(&argument); + break; } - break; /* Modify Nb Iterations */ case 'i': - if ((argument[1] >='0') && (argument[1] <='9')) { - int iters = argument[1] - '0'; - BMK_SetNbIterations(iters); + { argument++; + BMK_SetNbIterations((int)readU32FromChar(&argument)); } break; /* Select compressibility of synthetic sample */ case 'P': - { U32 proba32 = 0; - while ((argument[1]>= '0') && (argument[1]<= '9')) { - proba32 *= 10; - proba32 += argument[1] - '0'; - argument++; - } - g_compressibility = (double)proba32 / 100.; + { argument++; + g_compressibility = (double)readU32FromChar(&argument) / 100.; } break; + case 'l': + { argument++; + cLevel = readU32FromChar(&argument); + cparams = ZSTD_getCParams(cLevel, 0, 0); + } + break; + + /* Unknown command */ default : return badusage(exename); @@ -630,10 +737,12 @@ int main(int argc, const char** argv) if (!input_filename) { input_filename=argument; filenamesStart=i; continue; } } + + if (filenamesStart==0) /* no input file */ - result = benchSample(benchNb); + result = benchSample(benchNb, cLevel, &cparams); else - result = benchFiles(argv+filenamesStart, argc-filenamesStart, benchNb); + result = benchFiles(argv+filenamesStart, argc-filenamesStart, benchNb, cLevel, &cparams); if (main_pause) { int unused; printf("press enter...\n"); unused = getchar(); (void)unused; } From ed1a42987301ada4968c6acd86dafe56e8d10605 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 27 Jun 2018 16:57:28 -0700 Subject: [PATCH 012/372] test multi-lines travis yaml file --- .travis.yml | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/.travis.yml b/.travis.yml index b7099c24f..20c16fa90 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,28 +10,32 @@ addons: matrix: include: # Ubuntu 14.04 - - env: Cmd='make gcc6install && CC=gcc-6 make -j all && make clean && CC=gcc-6 make clean uasan-test-zstd' + - env: Cmd='make gcc6install && CC=gcc-6 make -j all \ + && make clean && CC=gcc-6 make clean uasan-test-zstd' - env: Cmd='make gcc6install libc6install && CC=gcc-6 make clean uasan-test-zstd32' - env: Cmd='make gcc7install && CC=gcc-7 make clean uasan-test-zstd' - env: Cmd='make clang38install && CC=clang-3.8 make clean msan-test-zstd' - env: Cmd='make gcc6install && CC=gcc-6 make clean uasan-fuzztest' - - env: Cmd='make gcc6install libc6install && CC=gcc-6 CFLAGS=-m32 make clean uasan-fuzztest' + - env: Cmd='make gcc6install libc6install \ + && make clean && CC=gcc-6 CFLAGS=-m32 make uasan-fuzztest' - env: Cmd='make clang38install && CC=clang-3.8 make clean msan-fuzztest' - env: Cmd='make clang38install && CC=clang-3.8 make clean tsan-test-zstream' - - env: Cmd='make -C tests test-fuzzer-stackmode' - - - env: Cmd='make valgrindinstall && make -C tests clean valgrindTest' - - env: Cmd='make arminstall && make armfuzz' - env: Cmd='make arminstall && make aarch64fuzz' - - env: Cmd='make ppcinstall && make ppcfuzz' - env: Cmd='make ppcinstall && make ppc64fuzz' - - env: Cmd='make -j uasanregressiontest && make clean && make -j msanregressiontest' - - env: Cmd='make lz4install && make -C tests test-lz4 test-pool && make clean && bash tests/libzstd_partial_builds.sh' + - env: Cmd='make -j uasanregressiontest \ + && make clean && make -j msanregressiontest' + + - env: Cmd='make valgrindinstall && make -C tests clean valgrindTest \ + && make clean && make -C tests test-fuzzer-stackmode' + + - env: Cmd='make lz4install && make -C tests test-lz4 \ + && make clean && make -C tests test-pool \ + && make clean && bash tests/libzstd_partial_builds.sh' # tag-specific test - if: tag =~ ^v[0-9]\.[0-9] @@ -44,6 +48,7 @@ branches: only: - dev - master + - travisTest script: - JOB_NUMBER=$(echo $TRAVIS_JOB_NUMBER | sed -e 's:[0-9][0-9]*\.\(.*\):\1:') From ab1ebd6578f9b1528382eaa948374b9eed490034 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 27 Jun 2018 19:29:18 -0700 Subject: [PATCH 013/372] removed \ at end of line --- .travis.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index 20c16fa90..566f3aa4e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,14 +10,14 @@ addons: matrix: include: # Ubuntu 14.04 - - env: Cmd='make gcc6install && CC=gcc-6 make -j all \ + - env: Cmd='make gcc6install && CC=gcc-6 make -j all && make clean && CC=gcc-6 make clean uasan-test-zstd' - env: Cmd='make gcc6install libc6install && CC=gcc-6 make clean uasan-test-zstd32' - env: Cmd='make gcc7install && CC=gcc-7 make clean uasan-test-zstd' - env: Cmd='make clang38install && CC=clang-3.8 make clean msan-test-zstd' - env: Cmd='make gcc6install && CC=gcc-6 make clean uasan-fuzztest' - - env: Cmd='make gcc6install libc6install \ + - env: Cmd='make gcc6install libc6install && make clean && CC=gcc-6 CFLAGS=-m32 make uasan-fuzztest' - env: Cmd='make clang38install && CC=clang-3.8 make clean msan-fuzztest' - env: Cmd='make clang38install && CC=clang-3.8 make clean tsan-test-zstream' @@ -27,14 +27,14 @@ matrix: - env: Cmd='make ppcinstall && make ppcfuzz' - env: Cmd='make ppcinstall && make ppc64fuzz' - - env: Cmd='make -j uasanregressiontest \ + - env: Cmd='make -j uasanregressiontest && make clean && make -j msanregressiontest' - - env: Cmd='make valgrindinstall && make -C tests clean valgrindTest \ + - env: Cmd='make valgrindinstall && make -C tests clean valgrindTest && make clean && make -C tests test-fuzzer-stackmode' - - env: Cmd='make lz4install && make -C tests test-lz4 \ - && make clean && make -C tests test-pool \ + - env: Cmd='make lz4install && make -C tests test-lz4 + && make clean && make -C tests test-pool && make clean && bash tests/libzstd_partial_builds.sh' # tag-specific test From 0ef06f2e8aa814ae0de80543cae3398aabf1d358 Mon Sep 17 00:00:00 2001 From: Jennifer Liu Date: Fri, 29 Jun 2018 12:33:34 -0700 Subject: [PATCH 014/372] Split samples into train and test sets --- lib/dictBuilder/cover.c | 69 ++++++++++++++++++++++++++++++++--------- lib/dictBuilder/zdict.h | 1 + 2 files changed, 55 insertions(+), 15 deletions(-) diff --git a/lib/dictBuilder/cover.c b/lib/dictBuilder/cover.c index 448f71372..c03ae9a11 100644 --- a/lib/dictBuilder/cover.c +++ b/lib/dictBuilder/cover.c @@ -39,6 +39,7 @@ * Constants ***************************************/ #define COVER_MAX_SAMPLES_SIZE (sizeof(size_t) == 8 ? ((U32)-1) : ((U32)1 GB)) +#define DEFAULT_SPLITPOINT 0.8 /*-************************************* * Console display @@ -203,6 +204,8 @@ typedef struct { size_t *offsets; const size_t *samplesSizes; size_t nbSamples; + size_t nbTrainSamples; + size_t nbTestSamples; U32 *suffix; size_t suffixSize; U32 *freqs; @@ -220,10 +223,10 @@ static COVER_ctx_t *g_ctx = NULL; /** * Returns the sum of the sample sizes. */ -static size_t COVER_sum(const size_t *samplesSizes, unsigned nbSamples) { +static size_t COVER_sum(const size_t *samplesSizes, unsigned firstSample, unsigned lastSample) { size_t sum = 0; size_t i; - for (i = 0; i < nbSamples; ++i) { + for (i = firstSample; i < lastSample; ++i) { sum += samplesSizes[i]; } return sum; @@ -494,6 +497,10 @@ static int COVER_checkParameters(ZDICT_cover_params_t parameters, if (parameters.d > parameters.k) { return 0; } + /* 0 < splitPoint < 1 */ + if (parameters.splitPoint <= 0 || parameters.splitPoint > 1){ + return 0; + } return 1; } @@ -531,9 +538,10 @@ static void COVER_ctx_destroy(COVER_ctx_t *ctx) { */ static int COVER_ctx_init(COVER_ctx_t *ctx, const void *samplesBuffer, const size_t *samplesSizes, unsigned nbSamples, - unsigned d) { + unsigned d, double splitPoint) { const BYTE *const samples = (const BYTE *)samplesBuffer; - const size_t totalSamplesSize = COVER_sum(samplesSizes, nbSamples); + const unsigned first = 0; + const size_t totalSamplesSize = COVER_sum(samplesSizes, first, nbSamples); /* Checks */ if (totalSamplesSize < MAX(d, sizeof(U64)) || totalSamplesSize >= (size_t)COVER_MAX_SAMPLES_SIZE) { @@ -541,15 +549,38 @@ static int COVER_ctx_init(COVER_ctx_t *ctx, const void *samplesBuffer, (U32)(totalSamplesSize>>20), (COVER_MAX_SAMPLES_SIZE >> 20)); return 0; } + /* Split samples into testing and training sets */ + const unsigned nbTrainSamples = nbSamples * splitPoint; + const unsigned nbTestSamples = nbSamples - nbTrainSamples; + /* Check if there's training sample */ + if (nbTrainSamples < 1) { + DISPLAYLEVEL(1, "Total number of training samples is %u and is invalid.", nbTrainSamples); + DISPLAYLEVEL(1, "splitPoint is %i", (int)(splitPoint*100)); + DISPLAYLEVEL(1, "nbSamples is %u", nbSamples); + return 0; + } + /* Check if there's testing sample when splitPoint is nonzero */ + if (nbTestSamples < 1 && splitPoint != 1.0) { + DISPLAYLEVEL(1, "Total number of testing samples is %u and is invalid.", nbTestSamples); + DISPLAYLEVEL(1, "splitPoint is %i", (int)(splitPoint*100)); + DISPLAYLEVEL(1, "nbSamples is %u", nbSamples); + return 0; + } + const size_t trainingSamplesSize = COVER_sum(samplesSizes, first, nbTrainSamples); + const size_t testSamplesSize = COVER_sum(samplesSizes, nbTrainSamples, nbSamples); /* Zero the context */ memset(ctx, 0, sizeof(*ctx)); - DISPLAYLEVEL(2, "Training on %u samples of total size %u\n", nbSamples, - (U32)totalSamplesSize); + DISPLAYLEVEL(2, "Training on %u samples of total size %u\n", nbTrainSamples, + (U32)trainingSamplesSize); + DISPLAYLEVEL(2, "Testing on %u samples of total size %u\n", nbTestSamples, + (U32)testSamplesSize); ctx->samples = samples; ctx->samplesSizes = samplesSizes; ctx->nbSamples = nbSamples; + ctx->nbTrainSamples = nbTrainSamples; + ctx->nbTestSamples = nbTestSamples; /* Partial suffix array */ - ctx->suffixSize = totalSamplesSize - MAX(d, sizeof(U64)) + 1; + ctx->suffixSize = trainingSamplesSize - MAX(d, sizeof(U64)) + 1; ctx->suffix = (U32 *)malloc(ctx->suffixSize * sizeof(U32)); /* Maps index to the dmerID */ ctx->dmerAt = (U32 *)malloc(ctx->suffixSize * sizeof(U32)); @@ -563,7 +594,7 @@ static int COVER_ctx_init(COVER_ctx_t *ctx, const void *samplesBuffer, ctx->freqs = NULL; ctx->d = d; - /* Fill offsets from the samlesSizes */ + /* Fill offsets from the samplesSizes */ { U32 i; ctx->offsets[0] = 0; @@ -665,7 +696,7 @@ ZDICTLIB_API size_t ZDICT_trainFromBuffer_cover( BYTE* const dict = (BYTE*)dictBuffer; COVER_ctx_t ctx; COVER_map_t activeDmers; - + parameters.splitPoint = 1.0; /* Initialize global data */ g_displayLevel = parameters.zParams.notificationLevel; /* Checks */ @@ -683,8 +714,9 @@ ZDICTLIB_API size_t ZDICT_trainFromBuffer_cover( return ERROR(dstSize_tooSmall); } /* Initialize context and activeDmers */ + const double all = 1.0; if (!COVER_ctx_init(&ctx, samplesBuffer, samplesSizes, nbSamples, - parameters.d)) { + parameters.d, all)) { return ERROR(GENERIC); } if (!COVER_map_init(&activeDmers, parameters.k - parameters.d + 1)) { @@ -839,7 +871,7 @@ typedef struct COVER_tryParameters_data_s { } COVER_tryParameters_data_t; /** - * Tries a set of parameters and upates the COVER_best_t with the results. + * Tries a set of parameters and updates the COVER_best_t with the results. * This function is thread safe if zstd is compiled with multithreaded support. * It takes its parameters as an *OWNING* opaque pointer to support threading. */ @@ -870,7 +902,7 @@ static void COVER_tryParameters(void *opaque) { dictBufferCapacity, parameters); dictBufferCapacity = ZDICT_finalizeDictionary( dict, dictBufferCapacity, dict + tail, dictBufferCapacity - tail, - ctx->samples, ctx->samplesSizes, (unsigned)ctx->nbSamples, + ctx->samples, ctx->samplesSizes, (unsigned)ctx->nbTrainSamples, parameters.zParams); if (ZDICT_isError(dictBufferCapacity)) { DISPLAYLEVEL(1, "Failed to finalize dictionary\n"); @@ -889,7 +921,7 @@ static void COVER_tryParameters(void *opaque) { /* Allocate dst with enough space to compress the maximum sized sample */ { size_t maxSampleSize = 0; - for (i = 0; i < ctx->nbSamples; ++i) { + for (i = ctx->nbTrainSamples; i < ctx->nbSamples; ++i) { maxSampleSize = MAX(ctx->samplesSizes[i], maxSampleSize); } dstCapacity = ZSTD_compressBound(maxSampleSize); @@ -904,7 +936,7 @@ static void COVER_tryParameters(void *opaque) { } /* Compress each sample and sum their sizes (or error) */ totalCompressedSize = dictBufferCapacity; - for (i = 0; i < ctx->nbSamples; ++i) { + for (i = ctx->nbTrainSamples; i < ctx->nbSamples; ++i) { const size_t size = ZSTD_compress_usingCDict( cctx, dst, dstCapacity, ctx->samples + ctx->offsets[i], ctx->samplesSizes[i], cdict); @@ -941,6 +973,8 @@ ZDICTLIB_API size_t ZDICT_optimizeTrainFromBuffer_cover( ZDICT_cover_params_t *parameters) { /* constants */ const unsigned nbThreads = parameters->nbThreads; + const double splitPoint = + parameters->splitPoint == 0 ? DEFAULT_SPLITPOINT : parameters->splitPoint; const unsigned kMinD = parameters->d == 0 ? 6 : parameters->d; const unsigned kMaxD = parameters->d == 0 ? 8 : parameters->d; const unsigned kMinK = parameters->k == 0 ? 50 : parameters->k; @@ -958,6 +992,10 @@ ZDICTLIB_API size_t ZDICT_optimizeTrainFromBuffer_cover( POOL_ctx *pool = NULL; /* Checks */ + if (splitPoint <= 0 || splitPoint >= 1) { + LOCALDISPLAYLEVEL(displayLevel, 1, "Incorrect parameters\n"); + return ERROR(GENERIC); + } if (kMinK < kMaxD || kMaxK < kMinK) { LOCALDISPLAYLEVEL(displayLevel, 1, "Incorrect parameters\n"); return ERROR(GENERIC); @@ -988,7 +1026,7 @@ ZDICTLIB_API size_t ZDICT_optimizeTrainFromBuffer_cover( /* Initialize the context for this value of d */ COVER_ctx_t ctx; LOCALDISPLAYLEVEL(displayLevel, 3, "d=%u\n", d); - if (!COVER_ctx_init(&ctx, samplesBuffer, samplesSizes, nbSamples, d)) { + if (!COVER_ctx_init(&ctx, samplesBuffer, samplesSizes, nbSamples, d, splitPoint)) { LOCALDISPLAYLEVEL(displayLevel, 1, "Failed to initialize context\n"); COVER_best_destroy(&best); POOL_free(pool); @@ -1013,6 +1051,7 @@ ZDICTLIB_API size_t ZDICT_optimizeTrainFromBuffer_cover( data->parameters = *parameters; data->parameters.k = k; data->parameters.d = d; + data->parameters.splitPoint = splitPoint; data->parameters.steps = kSteps; data->parameters.zParams.notificationLevel = g_displayLevel; /* Check the parameters */ diff --git a/lib/dictBuilder/zdict.h b/lib/dictBuilder/zdict.h index ad459c2d7..45d78b05f 100644 --- a/lib/dictBuilder/zdict.h +++ b/lib/dictBuilder/zdict.h @@ -86,6 +86,7 @@ typedef struct { unsigned d; /* dmer size : constraint: 0 < d <= k : Reasonable range [6, 16] */ unsigned steps; /* Number of steps : Only used for optimization : 0 means default (32) : Higher means more parameters checked */ unsigned nbThreads; /* Number of threads : constraint: 0 < nbThreads : 1 means single-threaded : Only used for optimization : Ignored if ZSTD_MULTITHREAD is not defined */ + double splitPoint; /* Percentage of samples used for training: the first nbSamples * splitPoint samples will be used to training */ ZDICT_params_t zParams; } ZDICT_cover_params_t; From 712a9fd9721c314f4b0238577d803b012845f6d2 Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Fri, 29 Jun 2018 15:33:44 -0400 Subject: [PATCH 015/372] Allow Invoking `zstd --list` When `stdin` is not a `tty` Also now returns an error when no inputs are given. New proposed behavior: ``` felix@odin:~/prog/zstd (list-stdin-check)$ ./zstd -l; echo $? No files given 1 felix@odin:~/prog/zstd (list-stdin-check)$ ./zstd -l Makefile.zst; echo $? Frames Skips Compressed Uncompressed Ratio Check Filename 1 0 3.08 KB 10.92 KB 3.544 XXH64 Makefile.zst 0 felix@odin:~/prog/zstd (list-stdin-check)$ ./zstd -l Date: Fri, 29 Jun 2018 12:47:03 -0700 Subject: [PATCH 016/372] Fix splitPoint floating point comparison problem --- lib/dictBuilder/cover.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/lib/dictBuilder/cover.c b/lib/dictBuilder/cover.c index c03ae9a11..dbb90c1f0 100644 --- a/lib/dictBuilder/cover.c +++ b/lib/dictBuilder/cover.c @@ -714,9 +714,8 @@ ZDICTLIB_API size_t ZDICT_trainFromBuffer_cover( return ERROR(dstSize_tooSmall); } /* Initialize context and activeDmers */ - const double all = 1.0; if (!COVER_ctx_init(&ctx, samplesBuffer, samplesSizes, nbSamples, - parameters.d, all)) { + parameters.d, parameters.splitPoint)) { return ERROR(GENERIC); } if (!COVER_map_init(&activeDmers, parameters.k - parameters.d + 1)) { @@ -974,7 +973,7 @@ ZDICTLIB_API size_t ZDICT_optimizeTrainFromBuffer_cover( /* constants */ const unsigned nbThreads = parameters->nbThreads; const double splitPoint = - parameters->splitPoint == 0 ? DEFAULT_SPLITPOINT : parameters->splitPoint; + parameters->splitPoint == 0.0 ? DEFAULT_SPLITPOINT : parameters->splitPoint; const unsigned kMinD = parameters->d == 0 ? 6 : parameters->d; const unsigned kMaxD = parameters->d == 0 ? 8 : parameters->d; const unsigned kMinK = parameters->k == 0 ? 50 : parameters->k; From 8e7bdc18d62632adcee029b2f8f5013d11549dd7 Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Fri, 29 Jun 2018 16:31:22 -0400 Subject: [PATCH 017/372] Fix Tests of `--list` Behavior with `stdin` --- tests/playTests.sh | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/tests/playTests.sh b/tests/playTests.sh index 09a7377f2..aa5535d59 100755 --- a/tests/playTests.sh +++ b/tests/playTests.sh @@ -731,8 +731,14 @@ $ECHO "\n===> zstd --list/-l error detection tests " ! $ZSTD -lv tmp1* ! $ZSTD --list -v tmp2 tmp12.zst -$ECHO "\n===> zstd --list/-l exits 1 when stdin is piped in" -! echo "piped STDIN" | $ZSTD --list +$ECHO "\n===> zstd --list/-l errors when presented with stdin / no files" +! $ZSTD -l +! $ZSTD -l - +! $ZSTD -l < tmp1.zst +! $ZSTD -l - < tmp1.zst +! $ZSTD -l - tmp1.zst +! $ZSTD -l - tmp1.zst < tmp1.zst +$ZSTD -l tmp1.zst < tmp1.zst # but doesn't error just because stdin is not a tty $ECHO "\n===> zstd --list/-l test with null files " ./datagen -g0 > tmp5 From e22d024e897febf78da7f1a6af0f7e16da17622f Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Fri, 29 Jun 2018 16:31:59 -0400 Subject: [PATCH 018/372] Make One Travis CI Run Run Tests With Non-TTY `stdin` --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index b7099c24f..80406064d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,7 +10,7 @@ addons: matrix: include: # Ubuntu 14.04 - - env: Cmd='make gcc6install && CC=gcc-6 make -j all && make clean && CC=gcc-6 make clean uasan-test-zstd' + - env: Cmd='make gcc6install && CC=gcc-6 make -j all && make clean && CC=gcc-6 make clean uasan-test-zstd Date: Fri, 29 Jun 2018 15:38:08 -0700 Subject: [PATCH 019/372] Another fix to comparator --- lib/dictBuilder/cover.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/dictBuilder/cover.c b/lib/dictBuilder/cover.c index dbb90c1f0..4dead220a 100644 --- a/lib/dictBuilder/cover.c +++ b/lib/dictBuilder/cover.c @@ -560,7 +560,7 @@ static int COVER_ctx_init(COVER_ctx_t *ctx, const void *samplesBuffer, return 0; } /* Check if there's testing sample when splitPoint is nonzero */ - if (nbTestSamples < 1 && splitPoint != 1.0) { + if (nbTestSamples < 1 && splitPoint < 1.0) { DISPLAYLEVEL(1, "Total number of testing samples is %u and is invalid.", nbTestSamples); DISPLAYLEVEL(1, "splitPoint is %i", (int)(splitPoint*100)); DISPLAYLEVEL(1, "nbSamples is %u", nbSamples); @@ -973,7 +973,7 @@ ZDICTLIB_API size_t ZDICT_optimizeTrainFromBuffer_cover( /* constants */ const unsigned nbThreads = parameters->nbThreads; const double splitPoint = - parameters->splitPoint == 0.0 ? DEFAULT_SPLITPOINT : parameters->splitPoint; + parameters->splitPoint <= 0.0 ? DEFAULT_SPLITPOINT : parameters->splitPoint; const unsigned kMinD = parameters->d == 0 ? 6 : parameters->d; const unsigned kMaxD = parameters->d == 0 ? 8 : parameters->d; const unsigned kMinK = parameters->k == 0 ? 50 : parameters->k; From f9d19b83fb92f0d99433604253cb4a198f2b90d3 Mon Sep 17 00:00:00 2001 From: Jennifer Liu Date: Fri, 29 Jun 2018 15:46:56 -0700 Subject: [PATCH 020/372] Fix variable declaration problem --- lib/dictBuilder/cover.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/dictBuilder/cover.c b/lib/dictBuilder/cover.c index 4dead220a..997036ecf 100644 --- a/lib/dictBuilder/cover.c +++ b/lib/dictBuilder/cover.c @@ -542,6 +542,11 @@ static int COVER_ctx_init(COVER_ctx_t *ctx, const void *samplesBuffer, const BYTE *const samples = (const BYTE *)samplesBuffer; const unsigned first = 0; const size_t totalSamplesSize = COVER_sum(samplesSizes, first, nbSamples); + /* Split samples into testing and training sets */ + const unsigned nbTrainSamples = nbSamples * splitPoint; + const unsigned nbTestSamples = nbSamples - nbTrainSamples; + const size_t trainingSamplesSize = COVER_sum(samplesSizes, first, nbTrainSamples); + const size_t testSamplesSize = COVER_sum(samplesSizes, nbTrainSamples, nbSamples); /* Checks */ if (totalSamplesSize < MAX(d, sizeof(U64)) || totalSamplesSize >= (size_t)COVER_MAX_SAMPLES_SIZE) { @@ -549,9 +554,6 @@ static int COVER_ctx_init(COVER_ctx_t *ctx, const void *samplesBuffer, (U32)(totalSamplesSize>>20), (COVER_MAX_SAMPLES_SIZE >> 20)); return 0; } - /* Split samples into testing and training sets */ - const unsigned nbTrainSamples = nbSamples * splitPoint; - const unsigned nbTestSamples = nbSamples - nbTrainSamples; /* Check if there's training sample */ if (nbTrainSamples < 1) { DISPLAYLEVEL(1, "Total number of training samples is %u and is invalid.", nbTrainSamples); @@ -566,8 +568,6 @@ static int COVER_ctx_init(COVER_ctx_t *ctx, const void *samplesBuffer, DISPLAYLEVEL(1, "nbSamples is %u", nbSamples); return 0; } - const size_t trainingSamplesSize = COVER_sum(samplesSizes, first, nbTrainSamples); - const size_t testSamplesSize = COVER_sum(samplesSizes, nbTrainSamples, nbSamples); /* Zero the context */ memset(ctx, 0, sizeof(*ctx)); DISPLAYLEVEL(2, "Training on %u samples of total size %u\n", nbTrainSamples, From 52fbbbcb6b906a6acd00ece3dfeed0a480d28751 Mon Sep 17 00:00:00 2001 From: Jennifer Liu Date: Fri, 29 Jun 2018 16:17:20 -0700 Subject: [PATCH 021/372] Explicitly cast double to unsigned --- lib/dictBuilder/cover.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/dictBuilder/cover.c b/lib/dictBuilder/cover.c index 997036ecf..53f3d79a8 100644 --- a/lib/dictBuilder/cover.c +++ b/lib/dictBuilder/cover.c @@ -543,7 +543,8 @@ static int COVER_ctx_init(COVER_ctx_t *ctx, const void *samplesBuffer, const unsigned first = 0; const size_t totalSamplesSize = COVER_sum(samplesSizes, first, nbSamples); /* Split samples into testing and training sets */ - const unsigned nbTrainSamples = nbSamples * splitPoint; + double tmp = (double)nbSamples * splitPoint; + const unsigned nbTrainSamples = (unsigned)tmp; const unsigned nbTestSamples = nbSamples - nbTrainSamples; const size_t trainingSamplesSize = COVER_sum(samplesSizes, first, nbTrainSamples); const size_t testSamplesSize = COVER_sum(samplesSizes, nbTrainSamples, nbSamples); From 328ec7e34c861dd99bfe4012652beef1bedb6e62 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Fri, 29 Jun 2018 16:52:21 -0700 Subject: [PATCH 022/372] fix `test-zstd` can be run with parallel compilation fix #1221 --- tests/Makefile | 30 ++++++++++++++---------------- 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/tests/Makefile b/tests/Makefile index 4bd43ea1f..4fffbc2d4 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -88,13 +88,8 @@ allnothread: fullbench fuzzer paramgrill datagen decodecorpus dll: fuzzer-dll zstreamtest-dll -zstd: - $(MAKE) -C $(PRGDIR) $@ MOREFLAGS+="$(DEBUGFLAGS)" - -zstd32: - $(MAKE) -C $(PRGDIR) $@ MOREFLAGS+="$(DEBUGFLAGS)" - -zstd-nolegacy: +PHONY: zstd zstd32 zstd-nolegacy # must be phony, only external makefile knows how to build them, or if they need an update +zstd zstd32 zstd-nolegacy: $(MAKE) -C $(PRGDIR) $@ MOREFLAGS+="$(DEBUGFLAGS)" gzstd: @@ -245,13 +240,14 @@ checkTag: checkTag.c $(ZSTDDIR)/zstd.h clean: $(MAKE) -C $(ZSTDDIR) clean + $(MAKE) -C $(PRGDIR) clean @$(RM) -fR $(TESTARTEFACT) @$(RM) -f core *.o tmp* result* *.gcda dictionary *.zst \ $(PRGDIR)/zstd$(EXT) $(PRGDIR)/zstd32$(EXT) \ fullbench$(EXT) fullbench32$(EXT) \ fullbench-lib$(EXT) fullbench-dll$(EXT) \ fuzzer$(EXT) fuzzer32$(EXT) zbufftest$(EXT) zbufftest32$(EXT) \ - fuzzer-dll$(EXT) zstreamtest-dll$(EXT) zbufftest-dll$(EXT)\ + fuzzer-dll$(EXT) zstreamtest-dll$(EXT) zbufftest-dll$(EXT) \ zstreamtest$(EXT) zstreamtest32$(EXT) \ datagen$(EXT) paramgrill$(EXT) roundTripCrash$(EXT) longmatch$(EXT) \ symbols$(EXT) invalidDictionaries$(EXT) legacy$(EXT) poolTests$(EXT) \ @@ -301,11 +297,6 @@ endif list: @$(MAKE) -pRrq -f $(lastword $(MAKEFILE_LIST)) : 2>/dev/null | awk -v RS= -F: '/^# File/,/^# Finished Make data base/ {if ($$1 !~ "^[#.]") {print $$1}}' | sort | egrep -v -e '^[^[:alnum:]]' -e '^$@$$' | xargs -.PHONY: zstd-playTests -zstd-playTests: datagen - file $(ZSTD) - ZSTD="$(QEMU_SYS) $(ZSTD)" ./playTests.sh $(ZSTDRTTEST) - .PHONY: shortest shortest: ZSTDRTTEST= shortest: test-zstd @@ -323,14 +314,21 @@ test32: test-zstd32 test-fullbench32 test-fuzzer32 test-zstream32 test-all: test test32 valgrindTest test-decodecorpus-cli + +.PHONY: test-zstd test-zstd32 test-zstd-nolegacy test-zstd: ZSTD = $(PRGDIR)/zstd -test-zstd: zstd zstd-playTests +test-zstd: zstd test-zstd32: ZSTD = $(PRGDIR)/zstd32 -test-zstd32: zstd32 zstd-playTests +test-zstd32: zstd32 test-zstd-nolegacy: ZSTD = $(PRGDIR)/zstd-nolegacy -test-zstd-nolegacy: zstd-nolegacy zstd-playTests +test-zstd-nolegacy: zstd-nolegacy + +test-zstd test-zstd32 test-zstd-nolegacy: datagen + file $(ZSTD) + ZSTD="$(QEMU_SYS) $(ZSTD)" ./playTests.sh $(ZSTDRTTEST) + test-gzstd: gzstd $(PRGDIR)/zstd -f README.md test-zstd-speed.py From b5207aadfaa0769a63bda0837a662957cbcd1f55 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Fri, 29 Jun 2018 17:10:56 -0700 Subject: [PATCH 023/372] make build tests more unforgiving `-Werror` will ensure they fail if there is the slightest warning. fix a minor warning specific to `zstd_decompress` variant. --- Makefile | 2 +- programs/zstdcli.c | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 59af7a018..0f2fcc958 100644 --- a/Makefile +++ b/Makefile @@ -64,7 +64,7 @@ zlibwrapper: .PHONY: test test: - $(MAKE) -C $(PRGDIR) allVariants MOREFLAGS+="-g -DDEBUGLEVEL=1" + MOREFLAGS+="-g -DDEBUGLEVEL=1 -Werror" $(MAKE) -C $(PRGDIR) allVariants $(MAKE) -C $(TESTDIR) $@ .PHONY: shortest diff --git a/programs/zstdcli.c b/programs/zstdcli.c index be6204667..cc3ec26c6 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -441,7 +441,7 @@ int main(int argCount, const char* argv[]) #endif /* preset behaviors */ - if (exeNameMatch(programName, ZSTD_ZSTDMT)) nbWorkers=0; + if (exeNameMatch(programName, ZSTD_ZSTDMT)) nbWorkers=0, singleThread=0; if (exeNameMatch(programName, ZSTD_UNZSTD)) operation=zom_decompress; if (exeNameMatch(programName, ZSTD_CAT)) { operation=zom_decompress; forceStdout=1; FIO_overwriteMode(); outFileName=stdoutmark; g_displayLevel=1; } /* supports multiple formats */ if (exeNameMatch(programName, ZSTD_ZCAT)) { operation=zom_decompress; forceStdout=1; FIO_overwriteMode(); outFileName=stdoutmark; g_displayLevel=1; } /* behave like zcat, also supports multiple formats */ @@ -764,7 +764,7 @@ int main(int argCount, const char* argv[]) DISPLAYLEVEL(3, "Note: %d physical core(s) detected \n", nbWorkers); } #else - (void)singleThread; + (void)singleThread; (void)nbWorkers; #endif #ifdef UTIL_HAS_CREATEFILELIST From 348e5f77a95922f4bf2232df1bd220ce665cc369 Mon Sep 17 00:00:00 2001 From: Jennifer Liu Date: Fri, 29 Jun 2018 17:54:41 -0700 Subject: [PATCH 024/372] Add split=# to cli --- lib/dictBuilder/cover.c | 8 ++++---- programs/zstd.1.md | 5 ++++- programs/zstdcli.c | 8 +++++++- 3 files changed, 15 insertions(+), 6 deletions(-) diff --git a/lib/dictBuilder/cover.c b/lib/dictBuilder/cover.c index 53f3d79a8..a3195aa77 100644 --- a/lib/dictBuilder/cover.c +++ b/lib/dictBuilder/cover.c @@ -558,15 +558,15 @@ static int COVER_ctx_init(COVER_ctx_t *ctx, const void *samplesBuffer, /* Check if there's training sample */ if (nbTrainSamples < 1) { DISPLAYLEVEL(1, "Total number of training samples is %u and is invalid.", nbTrainSamples); - DISPLAYLEVEL(1, "splitPoint is %i", (int)(splitPoint*100)); - DISPLAYLEVEL(1, "nbSamples is %u", nbSamples); return 0; } /* Check if there's testing sample when splitPoint is nonzero */ if (nbTestSamples < 1 && splitPoint < 1.0) { DISPLAYLEVEL(1, "Total number of testing samples is %u and is invalid.", nbTestSamples); - DISPLAYLEVEL(1, "splitPoint is %i", (int)(splitPoint*100)); - DISPLAYLEVEL(1, "nbSamples is %u", nbSamples); + return 0; + } + if (nbTrainSamples + nbTestSamples != nbSamples) { + DISPLAYLEVEL(1, "nbTrainSamples plus nbTestSamples don't add up to nbSamples"); return 0; } /* Zero the context */ diff --git a/programs/zstd.1.md b/programs/zstd.1.md index 4b3818141..c45bdb386 100644 --- a/programs/zstd.1.md +++ b/programs/zstd.1.md @@ -223,11 +223,12 @@ Compression of small files similar to the sample set will be greatly improved. This compares favorably to 4 bytes default. However, it's up to the dictionary manager to not assign twice the same ID to 2 different dictionaries. -* `--train-cover[=k#,d=#,steps=#]`: +* `--train-cover[=k#,d=#,steps=#,split=#]`: Select parameters for the default dictionary builder algorithm named cover. If _d_ is not specified, then it tries _d_ = 6 and _d_ = 8. If _k_ is not specified, then it tries _steps_ values in the range [50, 2000]. If _steps_ is not specified, then the default value of 40 is used. + If _split_ is not specified, then the default value of 80 is used. Requires that _d_ <= _k_. Selects segments of size _k_ with highest score to put in the dictionary. @@ -249,6 +250,8 @@ Compression of small files similar to the sample set will be greatly improved. `zstd --train-cover=k=50 FILEs` + `zstd --train-cover=k=50,split=60 FILEs` + * `--train-legacy[=selectivity=#]`: Use legacy dictionary builder algorithm with the given dictionary _selectivity_ (default: 9). diff --git a/programs/zstdcli.c b/programs/zstdcli.c index ae8c9cba9..68404d660 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -278,14 +278,20 @@ static unsigned longCommandWArg(const char** stringPtr, const char* longCommand) static unsigned parseCoverParameters(const char* stringPtr, ZDICT_cover_params_t* params) { memset(params, 0, sizeof(*params)); + unsigned splitPercentage = 100; for (; ;) { if (longCommandWArg(&stringPtr, "k=")) { params->k = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; } if (longCommandWArg(&stringPtr, "d=")) { params->d = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; } if (longCommandWArg(&stringPtr, "steps=")) { params->steps = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; } + if (longCommandWArg(&stringPtr, "split=")) { + splitPercentage = readU32FromChar(&stringPtr); + params->splitPoint = (double)splitPercentage / 100.0; + if (stringPtr[0]==',') { stringPtr++; continue; } else break; + } return 0; } if (stringPtr[0] != 0) return 0; - DISPLAYLEVEL(4, "cover: k=%u\nd=%u\nsteps=%u\n", params->k, params->d, params->steps); + DISPLAYLEVEL(4, "cover: k=%u\nd=%u\nsteps=%u\nsplitPoint=%d\n", params->k, params->d, params->steps, splitPercentage); return 1; } From 84e8b2a3059eb6360cc2dc7aeda3e0961119ce15 Mon Sep 17 00:00:00 2001 From: Jennifer Liu Date: Fri, 29 Jun 2018 18:02:02 -0700 Subject: [PATCH 025/372] Fix another declaration issue --- programs/zstdcli.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/programs/zstdcli.c b/programs/zstdcli.c index 68404d660..74dc607a3 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -277,8 +277,8 @@ static unsigned longCommandWArg(const char** stringPtr, const char* longCommand) */ static unsigned parseCoverParameters(const char* stringPtr, ZDICT_cover_params_t* params) { - memset(params, 0, sizeof(*params)); unsigned splitPercentage = 100; + memset(params, 0, sizeof(*params)); for (; ;) { if (longCommandWArg(&stringPtr, "k=")) { params->k = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; } if (longCommandWArg(&stringPtr, "d=")) { params->d = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; } From 276988f7948d95d917584a08d1d17a634d396890 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Ketelaars?= Date: Sat, 30 Jun 2018 13:01:58 +0200 Subject: [PATCH 026/372] OpenBSD is unable to write to /dev/zero https://github.com/facebook/zstd/pull/1124 fixes an issue with GNU/Hurd being unable to write to /dev/zero. Implemented fix is writing to /dev/random instead. On OpenBSD a regular user is unable to write to /dev/random because of permissions set on this device. Result is failing a regression test. Proposed solution should work for all platforms. --- tests/playTests.sh | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/playTests.sh b/tests/playTests.sh index aa5535d59..d167cc289 100755 --- a/tests/playTests.sh +++ b/tests/playTests.sh @@ -48,6 +48,8 @@ fileRoundTripTest() { $DIFF -q tmp.md5.1 tmp.md5.2 } +UNAME=$(uname) + isTerminal=false if [ -t 0 ] && [ -t 1 ] then @@ -56,7 +58,10 @@ fi isWindows=false INTOVOID="/dev/null" -DEVDEVICE="/dev/random" +case "$UNAME" in + OpenBSD) DEVDEVICE="/dev/zero" ;; + *) DEVDEVICE="/dev/random" ;; +esac case "$OS" in Windows*) isWindows=true @@ -65,7 +70,6 @@ case "$OS" in ;; esac -UNAME=$(uname) case "$UNAME" in Darwin) MD5SUM="md5 -r" ;; FreeBSD) MD5SUM="gmd5sum" ;; From 527dbf89ecf2984b88887377eb99ffbc0ca95302 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Ketelaars?= Date: Sat, 30 Jun 2018 13:22:14 +0200 Subject: [PATCH 027/372] xz/lzma warning causes test to fail OpenBSD's port building infrastructure is able to build in a privilege separated mode. It uses a privilege drop model. Regression tests fail in this mode as xz/lzma is unable to set file group and errors out with: xz: tmp.xz: Cannot set the file group: Operation not permitted gmake[1]: *** [Makefile:307: zstd-playTests] Error 2 Actually it is not a xz/lzma error but a warning causing zstd's regression test to fail. Proposed fix is to have xz/lzma not set the exit status to 2 even if a condition worth a warning was detected (-Q flag). --- tests/playTests.sh | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/tests/playTests.sh b/tests/playTests.sh index d167cc289..c4b1489b7 100755 --- a/tests/playTests.sh +++ b/tests/playTests.sh @@ -548,16 +548,16 @@ $ZSTD --format=xz -V || LZMAMODE=0 if [ $LZMAMODE -eq 1 ]; then $ECHO "xz support detected" XZEXE=1 - xz -V && lzma -V || XZEXE=0 + xz -Q -V && lzma -Q -V || XZEXE=0 if [ $XZEXE -eq 1 ]; then $ECHO "Testing zstd xz and lzma support" ./datagen > tmp $ZSTD --format=lzma -f tmp $ZSTD --format=xz -f tmp - xz -t -v tmp.xz - xz -t -v tmp.lzma - xz -f -k tmp - lzma -f -k --lzma1 tmp + xz -Q -t -v tmp.xz + xz -Q -t -v tmp.lzma + xz -Q -f -k tmp + lzma -Q -f -k --lzma1 tmp $ZSTD -d -f -v tmp.xz $ZSTD -d -f -v tmp.lzma rm tmp* @@ -569,13 +569,13 @@ if [ $LZMAMODE -eq 1 ]; then $ECHO "Testing xz and lzma symlinks" ./datagen > tmp ./xz tmp - xz -d tmp.xz + xz -Q -d tmp.xz ./lzma tmp - lzma -d tmp.lzma + lzma -Q -d tmp.lzma $ECHO "Testing unxz and unlzma symlinks" - xz tmp + xz -Q tmp ./xz -d tmp.xz - lzma tmp + lzma -Q tmp ./lzma -d tmp.lzma rm xz unxz lzma unlzma rm tmp* From 8afcb8eea76fb534950cbd2eab88847d6afb1442 Mon Sep 17 00:00:00 2001 From: Jennifer Liu Date: Sun, 1 Jul 2018 19:59:37 -0700 Subject: [PATCH 028/372] Update documentation --- programs/README.md | 2 +- programs/dibio.c | 3 ++- programs/zstd.1 | 4 ++-- programs/zstdcli.c | 2 +- 4 files changed, 6 insertions(+), 5 deletions(-) diff --git a/programs/README.md b/programs/README.md index a308fccf9..2833875e5 100644 --- a/programs/README.md +++ b/programs/README.md @@ -150,7 +150,7 @@ Advanced arguments : Dictionary builder : --train ## : create a dictionary from a training set of files ---train-cover[=k=#,d=#,steps=#] : use the cover algorithm with optional args +--train-cover[=k=#,d=#,steps=#,split=#] : use the cover algorithm with optional args --train-legacy[=s=#] : use the legacy algorithm with selectivity (default: 9) -o file : `file` is dictionary name (default: dictionary) --maxdict=# : limit dictionary to specified size (default: 112640) diff --git a/programs/dibio.c b/programs/dibio.c index 112259ddc..5d1f6d6c4 100644 --- a/programs/dibio.c +++ b/programs/dibio.c @@ -323,7 +323,8 @@ int DiB_trainFromFiles(const char* dictFileName, unsigned maxDictSize, srcBuffer, sampleSizes, fs.nbSamples, coverParams); if (!ZDICT_isError(dictSize)) { - DISPLAYLEVEL(2, "k=%u\nd=%u\nsteps=%u\n", coverParams->k, coverParams->d, coverParams->steps); + unsigned splitPercentage = (unsigned)(coverParams->splitPoint * 100); + DISPLAYLEVEL(2, "k=%u\nd=%u\nsteps=%u\nsplit=%u\n", coverParams->k, coverParams->d, coverParams->steps, splitPercentage); } } else { dictSize = ZDICT_trainFromBuffer_cover(dictBuffer, maxDictSize, srcBuffer, diff --git a/programs/zstd.1 b/programs/zstd.1 index 507933c97..e1ebd297e 100644 --- a/programs/zstd.1 +++ b/programs/zstd.1 @@ -217,8 +217,8 @@ Split input files in blocks of size # (default: no split) A dictionary ID is a locally unique ID that a decoder can use to verify it is using the right dictionary\. By default, zstd will create a 4\-bytes random number ID\. It\'s possible to give a precise number instead\. Short numbers have an advantage : an ID < 256 will only need 1 byte in the compressed frame header, and an ID < 65536 will only need 2 bytes\. This compares favorably to 4 bytes default\. However, it\'s up to the dictionary manager to not assign twice the same ID to 2 different dictionaries\. . .TP -\fB\-\-train\-cover[=k#,d=#,steps=#]\fR -Select parameters for the default dictionary builder algorithm named cover\. If \fId\fR is not specified, then it tries \fId\fR = 6 and \fId\fR = 8\. If \fIk\fR is not specified, then it tries \fIsteps\fR values in the range [50, 2000]\. If \fIsteps\fR is not specified, then the default value of 40 is used\. Requires that \fId\fR <= \fIk\fR\. +\fB\-\-train\-cover[=k#,d=#,steps=#,split=#]\fR +Select parameters for the default dictionary builder algorithm named cover\. If \fId\fR is not specified, then it tries \fId\fR = 6 and \fId\fR = 8\. If \fIk\fR is not specified, then it tries \fIsteps\fR values in the range [50, 2000]\. If \fIsteps\fR is not specified, then the default value of 40 is used\. If \fIsplit\fR is not specified, then the default value of 80 is used\. Requires that \fId\fR <= \fIk\fR\. . .IP Selects segments of size \fIk\fR with highest score to put in the dictionary\. The score of a segment is computed by the sum of the frequencies of all the subsegments of size \fId\fR\. Generally \fId\fR should be in the range [6, 8], occasionally up to 16, but the algorithm will run faster with d <= \fI8\fR\. Good values for \fIk\fR vary widely based on the input data, but a safe range is [2 * \fId\fR, 2000]\. Supports multithreading if \fBzstd\fR is compiled with threading support\. diff --git a/programs/zstdcli.c b/programs/zstdcli.c index 74dc607a3..28bed2309 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -170,7 +170,7 @@ static int usage_advanced(const char* programName) DISPLAY( "\n"); DISPLAY( "Dictionary builder : \n"); DISPLAY( "--train ## : create a dictionary from a training set of files \n"); - DISPLAY( "--train-cover[=k=#,d=#,steps=#] : use the cover algorithm with optional args\n"); + DISPLAY( "--train-cover[=k=#,d=#,steps=#,split=#] : use the cover algorithm with optional args\n"); DISPLAY( "--train-legacy[=s=#] : use the legacy algorithm with selectivity (default: %u)\n", g_defaultSelectivityLevel); DISPLAY( " -o file : `file` is dictionary name (default: %s) \n", g_defaultDictName); DISPLAY( "--maxdict=# : limit dictionary to specified size (default: %u) \n", g_defaultMaxDictSize); From 87579d51ee5c86266f50a219806f694306527e1d Mon Sep 17 00:00:00 2001 From: Jon Turney Date: Sun, 1 Jul 2018 22:41:08 +0100 Subject: [PATCH 029/372] meson: fix build --- contrib/meson/meson.build | 2 ++ 1 file changed, 2 insertions(+) diff --git a/contrib/meson/meson.build b/contrib/meson/meson.build index 079c045a1..98c9b0293 100644 --- a/contrib/meson/meson.build +++ b/contrib/meson/meson.build @@ -18,6 +18,7 @@ libzstd_srcs = [ join_paths(common_dir, 'error_private.c'), join_paths(common_dir, 'xxhash.c'), join_paths(compress_dir, 'fse_compress.c'), + join_paths(compress_dir, 'hist.c'), join_paths(compress_dir, 'huf_compress.c'), join_paths(compress_dir, 'zstd_compress.c'), join_paths(compress_dir, 'zstd_fast.c'), @@ -130,6 +131,7 @@ test('fuzzer', fuzzer) if target_machine.system() != 'windows' paramgrill = executable('paramgrill', datagen_c, join_paths(tests_dir, 'paramgrill.c'), + join_paths(programs_dir, 'bench.c'), include_directories: test_includes, link_with: libzstd, dependencies: libm) From 1a14f8639c60146e5e9a4bcced3849c057ed4244 Mon Sep 17 00:00:00 2001 From: Jennifer Liu Date: Mon, 2 Jul 2018 11:37:04 -0700 Subject: [PATCH 030/372] Update COVER dictionary builder tests --- tests/playTests.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/playTests.sh b/tests/playTests.sh index 09a7377f2..985b12d2d 100755 --- a/tests/playTests.sh +++ b/tests/playTests.sh @@ -408,7 +408,7 @@ $ECHO "\n===> cover dictionary builder : advanced options " TESTFILE=../programs/zstdcli.c ./datagen > tmpDict $ECHO "- Create first dictionary" -$ZSTD --train-cover=k=46,d=8 *.c ../programs/*.c -o tmpDict +$ZSTD --train-cover=k=46,d=8,split=80 *.c ../programs/*.c -o tmpDict cp $TESTFILE tmp $ZSTD -f tmp -D tmpDict $ZSTD -d tmp.zst -D tmpDict -fo result @@ -422,6 +422,9 @@ cmp tmpDict tmpDict1 && die "dictionaries should have different ID !" $ECHO "- Create dictionary with size limit" $ZSTD --train-cover=steps=8 *.c ../programs/*.c -o tmpDict2 --maxdict=4K rm tmp* +$ECHO "- Compare size of dictionary from 90% training samples with 80% training samples" +$ZSTD --train-cover=split=90 -r *.c ../programs/*.c +$ZSTD --train-cover=split=80 -r *.c ../programs/*.c $ECHO "\n===> legacy dictionary builder " From 16e75e88048874dfe1e21964ad7067b0eebf0ef4 Mon Sep 17 00:00:00 2001 From: Jennifer Liu Date: Tue, 3 Jul 2018 12:07:06 -0700 Subject: [PATCH 031/372] Update minimal training sample size --- lib/dictBuilder/cover.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/dictBuilder/cover.c b/lib/dictBuilder/cover.c index a3195aa77..1fe8d89ee 100644 --- a/lib/dictBuilder/cover.c +++ b/lib/dictBuilder/cover.c @@ -555,8 +555,8 @@ static int COVER_ctx_init(COVER_ctx_t *ctx, const void *samplesBuffer, (U32)(totalSamplesSize>>20), (COVER_MAX_SAMPLES_SIZE >> 20)); return 0; } - /* Check if there's training sample */ - if (nbTrainSamples < 1) { + /* Check if there are at least 5 training samples */ + if (nbTrainSamples < 5) { DISPLAYLEVEL(1, "Total number of training samples is %u and is invalid.", nbTrainSamples); return 0; } From 0881184c89917f1c21ffb53b972cb42b3fb2780a Mon Sep 17 00:00:00 2001 From: Jennifer Liu Date: Tue, 3 Jul 2018 17:53:27 -0700 Subject: [PATCH 032/372] Some edits based on pull request comments --- lib/dictBuilder/cover.c | 13 ++++++------- programs/zstdcli.c | 7 ++++--- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/lib/dictBuilder/cover.c b/lib/dictBuilder/cover.c index 1fe8d89ee..7ac7eb1c3 100644 --- a/lib/dictBuilder/cover.c +++ b/lib/dictBuilder/cover.c @@ -225,7 +225,7 @@ static COVER_ctx_t *g_ctx = NULL; */ static size_t COVER_sum(const size_t *samplesSizes, unsigned firstSample, unsigned lastSample) { size_t sum = 0; - size_t i; + unsigned i; for (i = firstSample; i < lastSample; ++i) { sum += samplesSizes[i]; } @@ -540,13 +540,12 @@ static int COVER_ctx_init(COVER_ctx_t *ctx, const void *samplesBuffer, const size_t *samplesSizes, unsigned nbSamples, unsigned d, double splitPoint) { const BYTE *const samples = (const BYTE *)samplesBuffer; - const unsigned first = 0; - const size_t totalSamplesSize = COVER_sum(samplesSizes, first, nbSamples); + const unsigned kFirst = 0; + const size_t totalSamplesSize = COVER_sum(samplesSizes, kFirst, nbSamples); /* Split samples into testing and training sets */ - double tmp = (double)nbSamples * splitPoint; - const unsigned nbTrainSamples = (unsigned)tmp; + const unsigned nbTrainSamples = (unsigned)((double)nbSamples * splitPoint); const unsigned nbTestSamples = nbSamples - nbTrainSamples; - const size_t trainingSamplesSize = COVER_sum(samplesSizes, first, nbTrainSamples); + const size_t trainingSamplesSize = COVER_sum(samplesSizes, kFirst, nbTrainSamples); const size_t testSamplesSize = COVER_sum(samplesSizes, nbTrainSamples, nbSamples); /* Checks */ if (totalSamplesSize < MAX(d, sizeof(U64)) || @@ -560,7 +559,7 @@ static int COVER_ctx_init(COVER_ctx_t *ctx, const void *samplesBuffer, DISPLAYLEVEL(1, "Total number of training samples is %u and is invalid.", nbTrainSamples); return 0; } - /* Check if there's testing sample when splitPoint is nonzero */ + /* Check if there's testing sample when splitPoint is not 1.0 */ if (nbTestSamples < 1 && splitPoint < 1.0) { DISPLAYLEVEL(1, "Total number of testing samples is %u and is invalid.", nbTestSamples); return 0; diff --git a/programs/zstdcli.c b/programs/zstdcli.c index 28bed2309..5408d2a51 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -84,6 +84,7 @@ static U32 g_ldmMinMatch = 0; static U32 g_ldmHashEveryLog = LDM_PARAM_DEFAULT; static U32 g_ldmBucketSizeLog = LDM_PARAM_DEFAULT; +#define DEFAULT_SPLITPOINT 0.8 /*-************************************ * Display Macros @@ -277,21 +278,20 @@ static unsigned longCommandWArg(const char** stringPtr, const char* longCommand) */ static unsigned parseCoverParameters(const char* stringPtr, ZDICT_cover_params_t* params) { - unsigned splitPercentage = 100; memset(params, 0, sizeof(*params)); for (; ;) { if (longCommandWArg(&stringPtr, "k=")) { params->k = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; } if (longCommandWArg(&stringPtr, "d=")) { params->d = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; } if (longCommandWArg(&stringPtr, "steps=")) { params->steps = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; } if (longCommandWArg(&stringPtr, "split=")) { - splitPercentage = readU32FromChar(&stringPtr); + unsigned splitPercentage = readU32FromChar(&stringPtr); params->splitPoint = (double)splitPercentage / 100.0; if (stringPtr[0]==',') { stringPtr++; continue; } else break; } return 0; } if (stringPtr[0] != 0) return 0; - DISPLAYLEVEL(4, "cover: k=%u\nd=%u\nsteps=%u\nsplitPoint=%d\n", params->k, params->d, params->steps, splitPercentage); + DISPLAYLEVEL(4, "cover: k=%u\nd=%u\nsteps=%u\nsplit=%u\n", params->k, params->d, params->steps, (unsigned)(params->splitPoint * 100)); return 1; } @@ -316,6 +316,7 @@ static ZDICT_cover_params_t defaultCoverParams(void) memset(¶ms, 0, sizeof(params)); params.d = 8; params.steps = 4; + params.splitPoint = DEFAULT_SPLITPOINT; return params; } #endif From a085d1aae1dbd841baa8ed927465e4d686ccc213 Mon Sep 17 00:00:00 2001 From: Jennifer Liu Date: Thu, 5 Jul 2018 10:38:45 -0700 Subject: [PATCH 033/372] Allow splitPoint==1.0 (using all samples for both training and testing) --- lib/dictBuilder/cover.c | 22 ++++++++++++---------- tests/playTests.sh | 2 ++ 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/lib/dictBuilder/cover.c b/lib/dictBuilder/cover.c index 7ac7eb1c3..2c19c0052 100644 --- a/lib/dictBuilder/cover.c +++ b/lib/dictBuilder/cover.c @@ -543,10 +543,10 @@ static int COVER_ctx_init(COVER_ctx_t *ctx, const void *samplesBuffer, const unsigned kFirst = 0; const size_t totalSamplesSize = COVER_sum(samplesSizes, kFirst, nbSamples); /* Split samples into testing and training sets */ - const unsigned nbTrainSamples = (unsigned)((double)nbSamples * splitPoint); - const unsigned nbTestSamples = nbSamples - nbTrainSamples; - const size_t trainingSamplesSize = COVER_sum(samplesSizes, kFirst, nbTrainSamples); - const size_t testSamplesSize = COVER_sum(samplesSizes, nbTrainSamples, nbSamples); + const unsigned nbTrainSamples = splitPoint < 1.0 ? (unsigned)((double)nbSamples * splitPoint) : nbSamples; + const unsigned nbTestSamples = splitPoint < 1.0 ? nbSamples - nbTrainSamples : nbSamples; + const size_t trainingSamplesSize = splitPoint < 1.0 ? COVER_sum(samplesSizes, kFirst, nbTrainSamples) : totalSamplesSize; + const size_t testSamplesSize = splitPoint < 1.0 ? COVER_sum(samplesSizes, nbTrainSamples, nbSamples) : totalSamplesSize; /* Checks */ if (totalSamplesSize < MAX(d, sizeof(U64)) || totalSamplesSize >= (size_t)COVER_MAX_SAMPLES_SIZE) { @@ -559,12 +559,13 @@ static int COVER_ctx_init(COVER_ctx_t *ctx, const void *samplesBuffer, DISPLAYLEVEL(1, "Total number of training samples is %u and is invalid.", nbTrainSamples); return 0; } - /* Check if there's testing sample when splitPoint is not 1.0 */ - if (nbTestSamples < 1 && splitPoint < 1.0) { + /* Check if there's testing sample */ + if (nbTestSamples < 1) { DISPLAYLEVEL(1, "Total number of testing samples is %u and is invalid.", nbTestSamples); return 0; } - if (nbTrainSamples + nbTestSamples != nbSamples) { + /* Check if nbTrainSamples plus nbTestSamples add up to nbSamples when splitPoint is less than 1*/ + if (nbTrainSamples + nbTestSamples != nbSamples && splitPoint < 1.0) { DISPLAYLEVEL(1, "nbTrainSamples plus nbTestSamples don't add up to nbSamples"); return 0; } @@ -920,7 +921,8 @@ static void COVER_tryParameters(void *opaque) { /* Allocate dst with enough space to compress the maximum sized sample */ { size_t maxSampleSize = 0; - for (i = ctx->nbTrainSamples; i < ctx->nbSamples; ++i) { + i = parameters.splitPoint < 1.0 ? ctx->nbTrainSamples : 0; + for (; i < ctx->nbSamples; ++i) { maxSampleSize = MAX(ctx->samplesSizes[i], maxSampleSize); } dstCapacity = ZSTD_compressBound(maxSampleSize); @@ -973,7 +975,7 @@ ZDICTLIB_API size_t ZDICT_optimizeTrainFromBuffer_cover( /* constants */ const unsigned nbThreads = parameters->nbThreads; const double splitPoint = - parameters->splitPoint <= 0.0 ? DEFAULT_SPLITPOINT : parameters->splitPoint; + (parameters->splitPoint <= 0.0 || parameters->splitPoint > 1.0) ? DEFAULT_SPLITPOINT : parameters->splitPoint; const unsigned kMinD = parameters->d == 0 ? 6 : parameters->d; const unsigned kMaxD = parameters->d == 0 ? 8 : parameters->d; const unsigned kMinK = parameters->k == 0 ? 50 : parameters->k; @@ -991,7 +993,7 @@ ZDICTLIB_API size_t ZDICT_optimizeTrainFromBuffer_cover( POOL_ctx *pool = NULL; /* Checks */ - if (splitPoint <= 0 || splitPoint >= 1) { + if (splitPoint <= 0 || splitPoint > 1) { LOCALDISPLAYLEVEL(displayLevel, 1, "Incorrect parameters\n"); return ERROR(GENERIC); } diff --git a/tests/playTests.sh b/tests/playTests.sh index 985b12d2d..3e1531375 100755 --- a/tests/playTests.sh +++ b/tests/playTests.sh @@ -425,6 +425,8 @@ rm tmp* $ECHO "- Compare size of dictionary from 90% training samples with 80% training samples" $ZSTD --train-cover=split=90 -r *.c ../programs/*.c $ZSTD --train-cover=split=80 -r *.c ../programs/*.c +$ECHO "- Create dictionary using all samples for both training and testing" +$ZSTD --train-cover=split=100 -r *.c ../programs/*.c $ECHO "\n===> legacy dictionary builder " From bfad1af0317a6994e3b46cc3111bf3796b8e82ac Mon Sep 17 00:00:00 2001 From: Jennifer Liu Date: Thu, 5 Jul 2018 11:05:31 -0700 Subject: [PATCH 034/372] Update doc for split==100 --- programs/zstd.1 | 2 +- programs/zstd.1.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/programs/zstd.1 b/programs/zstd.1 index e1ebd297e..b63ef4f2a 100644 --- a/programs/zstd.1 +++ b/programs/zstd.1 @@ -218,7 +218,7 @@ A dictionary ID is a locally unique ID that a decoder can use to verify it is us . .TP \fB\-\-train\-cover[=k#,d=#,steps=#,split=#]\fR -Select parameters for the default dictionary builder algorithm named cover\. If \fId\fR is not specified, then it tries \fId\fR = 6 and \fId\fR = 8\. If \fIk\fR is not specified, then it tries \fIsteps\fR values in the range [50, 2000]\. If \fIsteps\fR is not specified, then the default value of 40 is used\. If \fIsplit\fR is not specified, then the default value of 80 is used\. Requires that \fId\fR <= \fIk\fR\. +Select parameters for the default dictionary builder algorithm named cover\. If \fId\fR is not specified, then it tries \fId\fR = 6 and \fId\fR = 8\. If \fIk\fR is not specified, then it tries \fIsteps\fR values in the range [50, 2000]\. If \fIsteps\fR is not specified, then the default value of 40 is used\. If \fIsplit\fR is not specified or \fIsplit\fR <= 0 or \fIsplit\fR > 100, then the default value of 80 is used\. Requires that \fId\fR <= \fIk\fR\. . .IP Selects segments of size \fIk\fR with highest score to put in the dictionary\. The score of a segment is computed by the sum of the frequencies of all the subsegments of size \fId\fR\. Generally \fId\fR should be in the range [6, 8], occasionally up to 16, but the algorithm will run faster with d <= \fI8\fR\. Good values for \fIk\fR vary widely based on the input data, but a safe range is [2 * \fId\fR, 2000]\. Supports multithreading if \fBzstd\fR is compiled with threading support\. diff --git a/programs/zstd.1.md b/programs/zstd.1.md index c45bdb386..47035f1c0 100644 --- a/programs/zstd.1.md +++ b/programs/zstd.1.md @@ -228,7 +228,7 @@ Compression of small files similar to the sample set will be greatly improved. If _d_ is not specified, then it tries _d_ = 6 and _d_ = 8. If _k_ is not specified, then it tries _steps_ values in the range [50, 2000]. If _steps_ is not specified, then the default value of 40 is used. - If _split_ is not specified, then the default value of 80 is used. + If _split_ is not specified or split <= 0 or split > 100, then the default value of 80 is used. Requires that _d_ <= _k_. Selects segments of size _k_ with highest score to put in the dictionary. From 0bbff012119464fe4facf9b94f7502286df5f56d Mon Sep 17 00:00:00 2001 From: Jennifer Liu Date: Thu, 5 Jul 2018 22:40:32 -0700 Subject: [PATCH 035/372] Fix testing parameter --- lib/dictBuilder/cover.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/dictBuilder/cover.c b/lib/dictBuilder/cover.c index 2c19c0052..5fd2c9c74 100644 --- a/lib/dictBuilder/cover.c +++ b/lib/dictBuilder/cover.c @@ -937,7 +937,8 @@ static void COVER_tryParameters(void *opaque) { } /* Compress each sample and sum their sizes (or error) */ totalCompressedSize = dictBufferCapacity; - for (i = ctx->nbTrainSamples; i < ctx->nbSamples; ++i) { + i = parameters.splitPoint < 1.0 ? ctx->nbTrainSamples : 0; + for (; i < ctx->nbSamples; ++i) { const size_t size = ZSTD_compress_usingCDict( cctx, dst, dstCapacity, ctx->samples + ctx->offsets[i], ctx->samplesSizes[i], cdict); From 015a00af0f5f61c5093d990fe9cf7ec3d78c8839 Mon Sep 17 00:00:00 2001 From: Jennifer Liu Date: Fri, 6 Jul 2018 14:24:18 -0700 Subject: [PATCH 036/372] Change cover_sum back to 2 parameters and fix splitPoint issues --- lib/dictBuilder/cover.c | 18 ++++++------------ lib/dictBuilder/zdict.h | 2 +- 2 files changed, 7 insertions(+), 13 deletions(-) diff --git a/lib/dictBuilder/cover.c b/lib/dictBuilder/cover.c index 5fd2c9c74..176c386c4 100644 --- a/lib/dictBuilder/cover.c +++ b/lib/dictBuilder/cover.c @@ -223,10 +223,10 @@ static COVER_ctx_t *g_ctx = NULL; /** * Returns the sum of the sample sizes. */ -static size_t COVER_sum(const size_t *samplesSizes, unsigned firstSample, unsigned lastSample) { +static size_t COVER_sum(const size_t *samplesSizes, unsigned nbSamples) { size_t sum = 0; unsigned i; - for (i = firstSample; i < lastSample; ++i) { + for (i = 0; i < nbSamples; ++i) { sum += samplesSizes[i]; } return sum; @@ -540,13 +540,12 @@ static int COVER_ctx_init(COVER_ctx_t *ctx, const void *samplesBuffer, const size_t *samplesSizes, unsigned nbSamples, unsigned d, double splitPoint) { const BYTE *const samples = (const BYTE *)samplesBuffer; - const unsigned kFirst = 0; - const size_t totalSamplesSize = COVER_sum(samplesSizes, kFirst, nbSamples); + const size_t totalSamplesSize = COVER_sum(samplesSizes, nbSamples); /* Split samples into testing and training sets */ const unsigned nbTrainSamples = splitPoint < 1.0 ? (unsigned)((double)nbSamples * splitPoint) : nbSamples; const unsigned nbTestSamples = splitPoint < 1.0 ? nbSamples - nbTrainSamples : nbSamples; - const size_t trainingSamplesSize = splitPoint < 1.0 ? COVER_sum(samplesSizes, kFirst, nbTrainSamples) : totalSamplesSize; - const size_t testSamplesSize = splitPoint < 1.0 ? COVER_sum(samplesSizes, nbTrainSamples, nbSamples) : totalSamplesSize; + const size_t trainingSamplesSize = splitPoint < 1.0 ? COVER_sum(samplesSizes, nbTrainSamples) : totalSamplesSize; + const size_t testSamplesSize = splitPoint < 1.0 ? COVER_sum(samplesSizes + nbTrainSamples, nbTestSamples) : totalSamplesSize; /* Checks */ if (totalSamplesSize < MAX(d, sizeof(U64)) || totalSamplesSize >= (size_t)COVER_MAX_SAMPLES_SIZE) { @@ -564,11 +563,6 @@ static int COVER_ctx_init(COVER_ctx_t *ctx, const void *samplesBuffer, DISPLAYLEVEL(1, "Total number of testing samples is %u and is invalid.", nbTestSamples); return 0; } - /* Check if nbTrainSamples plus nbTestSamples add up to nbSamples when splitPoint is less than 1*/ - if (nbTrainSamples + nbTestSamples != nbSamples && splitPoint < 1.0) { - DISPLAYLEVEL(1, "nbTrainSamples plus nbTestSamples don't add up to nbSamples"); - return 0; - } /* Zero the context */ memset(ctx, 0, sizeof(*ctx)); DISPLAYLEVEL(2, "Training on %u samples of total size %u\n", nbTrainSamples, @@ -976,7 +970,7 @@ ZDICTLIB_API size_t ZDICT_optimizeTrainFromBuffer_cover( /* constants */ const unsigned nbThreads = parameters->nbThreads; const double splitPoint = - (parameters->splitPoint <= 0.0 || parameters->splitPoint > 1.0) ? DEFAULT_SPLITPOINT : parameters->splitPoint; + parameters->splitPoint <= 0.0 ? DEFAULT_SPLITPOINT : parameters->splitPoint; const unsigned kMinD = parameters->d == 0 ? 6 : parameters->d; const unsigned kMaxD = parameters->d == 0 ? 8 : parameters->d; const unsigned kMinK = parameters->k == 0 ? 50 : parameters->k; diff --git a/lib/dictBuilder/zdict.h b/lib/dictBuilder/zdict.h index 45d78b05f..8244c3bac 100644 --- a/lib/dictBuilder/zdict.h +++ b/lib/dictBuilder/zdict.h @@ -86,7 +86,7 @@ typedef struct { unsigned d; /* dmer size : constraint: 0 < d <= k : Reasonable range [6, 16] */ unsigned steps; /* Number of steps : Only used for optimization : 0 means default (32) : Higher means more parameters checked */ unsigned nbThreads; /* Number of threads : constraint: 0 < nbThreads : 1 means single-threaded : Only used for optimization : Ignored if ZSTD_MULTITHREAD is not defined */ - double splitPoint; /* Percentage of samples used for training: the first nbSamples * splitPoint samples will be used to training */ + double splitPoint; /* Percentage of samples used for training: the first nbSamples * splitPoint samples will be used to training, 0 means default (0.8) */ ZDICT_params_t zParams; } ZDICT_cover_params_t; From bbd78df59b1811045187bfa05d852b566d9b6934 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Fri, 6 Jul 2018 17:06:04 -0700 Subject: [PATCH 037/372] add build macro NO_PREFETCH prevent usage of prefetch intrinsic commands which are not supported by c2rust (see https://github.com/immunant/c2rust/issues/13) --- lib/common/compiler.h | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/lib/common/compiler.h b/lib/common/compiler.h index 366ed2b4b..595bac204 100644 --- a/lib/common/compiler.h +++ b/lib/common/compiler.h @@ -88,15 +88,20 @@ #endif #endif -/* prefetch */ -#if defined(_MSC_VER) && (defined(_M_X64) || defined(_M_I86)) /* _mm_prefetch() is not defined outside of x86/x64 */ -# include /* https://msdn.microsoft.com/fr-fr/library/84szxsww(v=vs.90).aspx */ -# define PREFETCH(ptr) _mm_prefetch((const char*)ptr, _MM_HINT_T0) -#elif defined(__GNUC__) && ( (__GNUC__ >= 4) || ( (__GNUC__ == 3) && (__GNUC_MINOR__ >= 1) ) ) -# define PREFETCH(ptr) __builtin_prefetch(ptr, 0, 0) -#else +/* prefetch + * can be disabled, by declaring NO_PREFETCH macro */ +#if defined(NO_PREFETCH) # define PREFETCH(ptr) /* disabled */ -#endif +#else +# if defined(_MSC_VER) && (defined(_M_X64) || defined(_M_I86)) /* _mm_prefetch() is not defined outside of x86/x64 */ +# include /* https://msdn.microsoft.com/fr-fr/library/84szxsww(v=vs.90).aspx */ +# define PREFETCH(ptr) _mm_prefetch((const char*)ptr, _MM_HINT_T0) +# elif defined(__GNUC__) && ( (__GNUC__ >= 4) || ( (__GNUC__ == 3) && (__GNUC_MINOR__ >= 1) ) ) +# define PREFETCH(ptr) __builtin_prefetch(ptr, 0, 0) +# else +# define PREFETCH(ptr) /* disabled */ +# endif +#endif /* NO_PREFETCH */ /* disable warnings */ #ifdef _MSC_VER /* Visual Studio */ From 7efabb2cf68fed6ebbd92dbecd653496e4e26a42 Mon Sep 17 00:00:00 2001 From: Jennifer Liu Date: Mon, 9 Jul 2018 12:26:53 -0700 Subject: [PATCH 038/372] Only make 0.0 default splitPoint --- lib/dictBuilder/cover.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/dictBuilder/cover.c b/lib/dictBuilder/cover.c index 176c386c4..3a97965cf 100644 --- a/lib/dictBuilder/cover.c +++ b/lib/dictBuilder/cover.c @@ -970,7 +970,7 @@ ZDICTLIB_API size_t ZDICT_optimizeTrainFromBuffer_cover( /* constants */ const unsigned nbThreads = parameters->nbThreads; const double splitPoint = - parameters->splitPoint <= 0.0 ? DEFAULT_SPLITPOINT : parameters->splitPoint; + parameters->splitPoint == 0.0 ? DEFAULT_SPLITPOINT : parameters->splitPoint; const unsigned kMinD = parameters->d == 0 ? 6 : parameters->d; const unsigned kMaxD = parameters->d == 0 ? 8 : parameters->d; const unsigned kMinK = parameters->k == 0 ? 50 : parameters->k; From 456f290e31cc81d436f15f17f15f711e0fcba47b Mon Sep 17 00:00:00 2001 From: Jennifer Liu Date: Mon, 9 Jul 2018 13:53:25 -0700 Subject: [PATCH 039/372] Change back to splitPoint<=0 --- lib/dictBuilder/cover.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/dictBuilder/cover.c b/lib/dictBuilder/cover.c index 3a97965cf..176c386c4 100644 --- a/lib/dictBuilder/cover.c +++ b/lib/dictBuilder/cover.c @@ -970,7 +970,7 @@ ZDICTLIB_API size_t ZDICT_optimizeTrainFromBuffer_cover( /* constants */ const unsigned nbThreads = parameters->nbThreads; const double splitPoint = - parameters->splitPoint == 0.0 ? DEFAULT_SPLITPOINT : parameters->splitPoint; + parameters->splitPoint <= 0.0 ? DEFAULT_SPLITPOINT : parameters->splitPoint; const unsigned kMinD = parameters->d == 0 ? 6 : parameters->d; const unsigned kMaxD = parameters->d == 0 ? 8 : parameters->d; const unsigned kMinK = parameters->k == 0 ? 50 : parameters->k; From 5021441d86ab366a70584a326eba44d9c9e5aaea Mon Sep 17 00:00:00 2001 From: Jennifer Liu Date: Tue, 10 Jul 2018 11:19:33 -0700 Subject: [PATCH 040/372] Change default splitPoint to 100 --- lib/dictBuilder/cover.c | 4 ++-- lib/dictBuilder/zdict.h | 2 +- programs/zstd.1 | 2 +- programs/zstd.1.md | 2 +- programs/zstdcli.c | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/dictBuilder/cover.c b/lib/dictBuilder/cover.c index 176c386c4..e32991652 100644 --- a/lib/dictBuilder/cover.c +++ b/lib/dictBuilder/cover.c @@ -39,7 +39,7 @@ * Constants ***************************************/ #define COVER_MAX_SAMPLES_SIZE (sizeof(size_t) == 8 ? ((U32)-1) : ((U32)1 GB)) -#define DEFAULT_SPLITPOINT 0.8 +#define DEFAULT_SPLITPOINT 1.0 /*-************************************* * Console display @@ -497,7 +497,7 @@ static int COVER_checkParameters(ZDICT_cover_params_t parameters, if (parameters.d > parameters.k) { return 0; } - /* 0 < splitPoint < 1 */ + /* 0 < splitPoint <= 1 */ if (parameters.splitPoint <= 0 || parameters.splitPoint > 1){ return 0; } diff --git a/lib/dictBuilder/zdict.h b/lib/dictBuilder/zdict.h index 8244c3bac..9357e40a6 100644 --- a/lib/dictBuilder/zdict.h +++ b/lib/dictBuilder/zdict.h @@ -86,7 +86,7 @@ typedef struct { unsigned d; /* dmer size : constraint: 0 < d <= k : Reasonable range [6, 16] */ unsigned steps; /* Number of steps : Only used for optimization : 0 means default (32) : Higher means more parameters checked */ unsigned nbThreads; /* Number of threads : constraint: 0 < nbThreads : 1 means single-threaded : Only used for optimization : Ignored if ZSTD_MULTITHREAD is not defined */ - double splitPoint; /* Percentage of samples used for training: the first nbSamples * splitPoint samples will be used to training, 0 means default (0.8) */ + double splitPoint; /* Percentage of samples used for training: the first nbSamples * splitPoint samples will be used to training, 0 means default (1.0) */ ZDICT_params_t zParams; } ZDICT_cover_params_t; diff --git a/programs/zstd.1 b/programs/zstd.1 index b63ef4f2a..3e9e29423 100644 --- a/programs/zstd.1 +++ b/programs/zstd.1 @@ -218,7 +218,7 @@ A dictionary ID is a locally unique ID that a decoder can use to verify it is us . .TP \fB\-\-train\-cover[=k#,d=#,steps=#,split=#]\fR -Select parameters for the default dictionary builder algorithm named cover\. If \fId\fR is not specified, then it tries \fId\fR = 6 and \fId\fR = 8\. If \fIk\fR is not specified, then it tries \fIsteps\fR values in the range [50, 2000]\. If \fIsteps\fR is not specified, then the default value of 40 is used\. If \fIsplit\fR is not specified or \fIsplit\fR <= 0 or \fIsplit\fR > 100, then the default value of 80 is used\. Requires that \fId\fR <= \fIk\fR\. +Select parameters for the default dictionary builder algorithm named cover\. If \fId\fR is not specified, then it tries \fId\fR = 6 and \fId\fR = 8\. If \fIk\fR is not specified, then it tries \fIsteps\fR values in the range [50, 2000]\. If \fIsteps\fR is not specified, then the default value of 40 is used\. If \fIsplit\fR is not specified or \fIsplit\fR <= 0 or \fIsplit\fR > 100, then the default value of 100 is used\. Requires that \fId\fR <= \fIk\fR\. . .IP Selects segments of size \fIk\fR with highest score to put in the dictionary\. The score of a segment is computed by the sum of the frequencies of all the subsegments of size \fId\fR\. Generally \fId\fR should be in the range [6, 8], occasionally up to 16, but the algorithm will run faster with d <= \fI8\fR\. Good values for \fIk\fR vary widely based on the input data, but a safe range is [2 * \fId\fR, 2000]\. Supports multithreading if \fBzstd\fR is compiled with threading support\. diff --git a/programs/zstd.1.md b/programs/zstd.1.md index 47035f1c0..df6f777df 100644 --- a/programs/zstd.1.md +++ b/programs/zstd.1.md @@ -228,7 +228,7 @@ Compression of small files similar to the sample set will be greatly improved. If _d_ is not specified, then it tries _d_ = 6 and _d_ = 8. If _k_ is not specified, then it tries _steps_ values in the range [50, 2000]. If _steps_ is not specified, then the default value of 40 is used. - If _split_ is not specified or split <= 0 or split > 100, then the default value of 80 is used. + If _split_ is not specified or split <= 0 or split > 100, then the default value of 100 is used. Requires that _d_ <= _k_. Selects segments of size _k_ with highest score to put in the dictionary. diff --git a/programs/zstdcli.c b/programs/zstdcli.c index 5408d2a51..a466a7ff3 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -84,7 +84,7 @@ static U32 g_ldmMinMatch = 0; static U32 g_ldmHashEveryLog = LDM_PARAM_DEFAULT; static U32 g_ldmBucketSizeLog = LDM_PARAM_DEFAULT; -#define DEFAULT_SPLITPOINT 0.8 +#define DEFAULT_SPLITPOINT 1.0 /*-************************************ * Display Macros From c1a7defee1f81b0f0a67875042e3a3621e6e0f97 Mon Sep 17 00:00:00 2001 From: Nick Terrell Date: Tue, 10 Jul 2018 15:07:36 -0700 Subject: [PATCH 041/372] Small fixes to zstd specification Update to keep in sync with the RFC. --- doc/zstd_compression_format.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/zstd_compression_format.md b/doc/zstd_compression_format.md index 62430e48f..0b79f959f 100644 --- a/doc/zstd_compression_format.md +++ b/doc/zstd_compression_format.md @@ -488,20 +488,20 @@ For values spanning several bytes, convention is __little-endian__. __`Size_Format` for `Raw_Literals_Block` and `RLE_Literals_Block`__ : `Size_Format` uses 1 _or_ 2 bits. -Its value is : `Size_Format = (Header[0]>>2) & 3` +Its value is : `Size_Format = (Literals_Section_Header[0]>>2) & 3` - `Size_Format` == 00 or 10 : `Size_Format` uses 1 bit. `Regenerated_Size` uses 5 bits (0-31). `Literals_Section_Header` uses 1 byte. - `Regenerated_Size = Header[0]>>3` + `Regenerated_Size = Literals_Section_Header[0]>>3` - `Size_Format` == 01 : `Size_Format` uses 2 bits. `Regenerated_Size` uses 12 bits (0-4095). `Literals_Section_Header` uses 2 bytes. - `Regenerated_Size = (Header[0]>>4) + (Header[1]<<4)` + `Regenerated_Size = (Literals_Section_Header[0]>>4) + (Literals_Section_Header[1]<<4)` - `Size_Format` == 11 : `Size_Format` uses 2 bits. `Regenerated_Size` uses 20 bits (0-1048575). `Literals_Section_Header` uses 3 bytes. - `Regenerated_Size = (Header[0]>>4) + (Header[1]<<4) + (Header[2]<<12)` + `Regenerated_Size = (Literals_Section_Header[0]>>4) + (Literals_Section_Header[1]<<4) + (Literals_Section_Header[2]<<12)` Only Stream1 is present for these cases. Note : it's allowed to represent a short value (for example `13`) From 7ca12a1455cc47c2f178fbd5b963fa8a1e4f0e71 Mon Sep 17 00:00:00 2001 From: "Dmitry V. Levin" Date: Wed, 11 Jul 2018 12:41:50 +0000 Subject: [PATCH 042/372] tests: use /dev/zero instead of /dev/random on all systems except GNU/Hurd https://github.com/facebook/zstd/pull/1124 broke the test suite on Linux. Looks like GNU/Hurd is the only operating system where /dev/random is available for writing by unprivileged processes. https://github.com/facebook/zstd/pull/1222 reverted the change introduced by https://github.com/facebook/zstd/pull/1124 for OpenBSD only, other operating systems need that change to be reverted, too. Fixes: 2dde9d5abaad ("Write to /dev/random for test") Complements: 276988f7948d ("OpenBSD is unable to write to /dev/zero") --- tests/playTests.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/playTests.sh b/tests/playTests.sh index c4b1489b7..65c661e17 100755 --- a/tests/playTests.sh +++ b/tests/playTests.sh @@ -59,8 +59,8 @@ fi isWindows=false INTOVOID="/dev/null" case "$UNAME" in - OpenBSD) DEVDEVICE="/dev/zero" ;; - *) DEVDEVICE="/dev/random" ;; + GNU) DEVDEVICE="/dev/random" ;; + *) DEVDEVICE="/dev/zero" ;; esac case "$OS" in Windows*) From 23d77c531e44fd4a4af2c3de2ba13dd6263f9767 Mon Sep 17 00:00:00 2001 From: Codecat Date: Wed, 11 Jul 2018 18:02:18 +0200 Subject: [PATCH 043/372] Added premake4/GENie script to contrib folder --- contrib/premake/premake4.lua | 6 +++ contrib/premake/zstd.lua | 79 ++++++++++++++++++++++++++++++++++++ 2 files changed, 85 insertions(+) create mode 100644 contrib/premake/premake4.lua create mode 100644 contrib/premake/zstd.lua diff --git a/contrib/premake/premake4.lua b/contrib/premake/premake4.lua new file mode 100644 index 000000000..6675e2e48 --- /dev/null +++ b/contrib/premake/premake4.lua @@ -0,0 +1,6 @@ +-- Include zstd.lua in your GENie or premake4 file, which exposes a project_zstd function +dofile('zstd.lua') + +solution 'example' + configurations { 'Debug', 'Release' } + project_zstd('../../lib/') diff --git a/contrib/premake/zstd.lua b/contrib/premake/zstd.lua new file mode 100644 index 000000000..4759dff76 --- /dev/null +++ b/contrib/premake/zstd.lua @@ -0,0 +1,79 @@ +-- This GENie/premake file copies the behavior of the Makefile in the lib folder. +-- Basic usage: project_zstd(ZSTD_DIR) + +function project_zstd(dir, compression, decompression, deprecated, dictbuilder, legacy) + if compression == nil then compression = true end + if decompression == nil then decompression = true end + if deprecated == nil then deprecated = false end + if dictbuilder == nil then dictbuilder = false end + + if legacy == nil then legacy = 0 end + + if compression then + dictbuilder = false + deprecated = false + end + + if decompression then + legacy = 0 + deprecated = false + end + + project 'zstd' + kind 'StaticLib' + language 'C' + + files { + dir .. 'zstd.h', + dir .. 'common/**.c', + dir .. 'common/**.h' + } + + if compression then + files { + dir .. 'compress/**.c', + dir .. 'compress/**.h' + } + end + + if decompression then + files { + dir .. 'decompress/**.c', + dir .. 'decompress/**.h' + } + end + + if dictbuilder then + files { + dir .. 'dictBuilder/**.c', + dir .. 'dictBuilder/**.h' + } + end + + if deprecated then + files { + dir .. 'deprecated/**.c', + dir .. 'deprecated/**.h' + } + end + + if legacy < 8 then + files { + dir .. 'legacy/zstd_v0' .. (legacy - 7) .. '.*' + } + else + includedirs { + dir .. 'legacy' + } + end + + includedirs { + dir, + dir .. 'common' + } + + defines { + 'XXH_NAMESPACE=ZSTD_', + 'ZSTD_LEGACY_SUPPORT=' .. legacy + } +end From 612b346ed51c8497e40c521c77808833361945ab Mon Sep 17 00:00:00 2001 From: Jennifer Liu Date: Wed, 11 Jul 2018 15:50:28 -0700 Subject: [PATCH 044/372] Add explanation for split=100 --- lib/dictBuilder/zdict.h | 2 +- programs/zstd.1 | 3 ++- programs/zstd.1.md | 4 +++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/lib/dictBuilder/zdict.h b/lib/dictBuilder/zdict.h index 9357e40a6..4094669d1 100644 --- a/lib/dictBuilder/zdict.h +++ b/lib/dictBuilder/zdict.h @@ -86,7 +86,7 @@ typedef struct { unsigned d; /* dmer size : constraint: 0 < d <= k : Reasonable range [6, 16] */ unsigned steps; /* Number of steps : Only used for optimization : 0 means default (32) : Higher means more parameters checked */ unsigned nbThreads; /* Number of threads : constraint: 0 < nbThreads : 1 means single-threaded : Only used for optimization : Ignored if ZSTD_MULTITHREAD is not defined */ - double splitPoint; /* Percentage of samples used for training: the first nbSamples * splitPoint samples will be used to training, 0 means default (1.0) */ + double splitPoint; /* Percentage of samples used for training: the first nbSamples * splitPoint samples will be used to training, the last nbSamples * (1 - splitPoint) samples will be used for testing, 0 means default (1.0), 1.0 when all samples are used for both training and testing */ ZDICT_params_t zParams; } ZDICT_cover_params_t; diff --git a/programs/zstd.1 b/programs/zstd.1 index 3e9e29423..9d1e45cfe 100644 --- a/programs/zstd.1 +++ b/programs/zstd.1 @@ -218,7 +218,8 @@ A dictionary ID is a locally unique ID that a decoder can use to verify it is us . .TP \fB\-\-train\-cover[=k#,d=#,steps=#,split=#]\fR -Select parameters for the default dictionary builder algorithm named cover\. If \fId\fR is not specified, then it tries \fId\fR = 6 and \fId\fR = 8\. If \fIk\fR is not specified, then it tries \fIsteps\fR values in the range [50, 2000]\. If \fIsteps\fR is not specified, then the default value of 40 is used\. If \fIsplit\fR is not specified or \fIsplit\fR <= 0 or \fIsplit\fR > 100, then the default value of 100 is used\. Requires that \fId\fR <= \fIk\fR\. +Select parameters for the default dictionary builder algorithm named cover\. If \fId\fR is not specified, then it tries \fId\fR = 6 and \fId\fR = 8\. If \fIk\fR is not specified, then it tries \fIsteps\fR values in the range [50, 2000]\. If \fIsteps\fR is not specified, then the default value of 40 is used\. If \fIsplit\fR is not specified or \fIsplit\fR <= 0, then the default value of 100 is used\. If \fIsplit\fR is 100, all input samples are used for both training and testing +to find optimal _d_ and _k_ to build dictionary.Requires that \fId\fR <= \fIk\fR\. . .IP Selects segments of size \fIk\fR with highest score to put in the dictionary\. The score of a segment is computed by the sum of the frequencies of all the subsegments of size \fId\fR\. Generally \fId\fR should be in the range [6, 8], occasionally up to 16, but the algorithm will run faster with d <= \fI8\fR\. Good values for \fIk\fR vary widely based on the input data, but a safe range is [2 * \fId\fR, 2000]\. Supports multithreading if \fBzstd\fR is compiled with threading support\. diff --git a/programs/zstd.1.md b/programs/zstd.1.md index df6f777df..7e21073d7 100644 --- a/programs/zstd.1.md +++ b/programs/zstd.1.md @@ -228,7 +228,7 @@ Compression of small files similar to the sample set will be greatly improved. If _d_ is not specified, then it tries _d_ = 6 and _d_ = 8. If _k_ is not specified, then it tries _steps_ values in the range [50, 2000]. If _steps_ is not specified, then the default value of 40 is used. - If _split_ is not specified or split <= 0 or split > 100, then the default value of 100 is used. + If _split_ is not specified or split <= 0, then the default value of 100 is used. Requires that _d_ <= _k_. Selects segments of size _k_ with highest score to put in the dictionary. @@ -238,6 +238,8 @@ Compression of small files similar to the sample set will be greatly improved. algorithm will run faster with d <= _8_. Good values for _k_ vary widely based on the input data, but a safe range is [2 * _d_, 2000]. + If _split_ is 100, all input samples are used for both training and testing + to find optimal _d_ and _k_ to build dictionary. Supports multithreading if `zstd` is compiled with threading support. Examples: From 6d222c437ca1f3b7420f414354643d3f11ba075a Mon Sep 17 00:00:00 2001 From: Nick Terrell Date: Thu, 12 Jul 2018 17:56:58 -0700 Subject: [PATCH 045/372] Set requestedParams in ZSTD_initCStream*() The correct parameters are used once, but once `ZSTD_resetCStream()` is called the default parameters (level 3) are used. Fix this by setting `requestedParams` in the `ZSTD_initCStream*()` functions. The added tests both fail before this patch and pass after. --- lib/compress/zstd_compress.c | 22 ++++++++++------------ tests/zstreamtest.c | 20 ++++++++++++++++++++ 2 files changed, 30 insertions(+), 12 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index c66862524..d659baf12 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -3332,9 +3332,11 @@ size_t ZSTD_CStreamOutSize(void) static size_t ZSTD_resetCStream_internal(ZSTD_CStream* cctx, const void* const dict, size_t const dictSize, ZSTD_dictContentType_e const dictContentType, const ZSTD_CDict* const cdict, - ZSTD_CCtx_params const params, unsigned long long const pledgedSrcSize) + ZSTD_CCtx_params params, unsigned long long const pledgedSrcSize) { DEBUGLOG(4, "ZSTD_resetCStream_internal"); + /* Finalize the compression parameters */ + params.cParams = ZSTD_getCParamsFromCCtxParams(¶ms, pledgedSrcSize, dictSize); /* params are supposed to be fully validated at this point */ assert(!ZSTD_isError(ZSTD_checkCParams(params.cParams))); assert(!((dict) && (cdict))); /* either dict or cdict, not both */ @@ -3363,7 +3365,6 @@ size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize) DEBUGLOG(4, "ZSTD_resetCStream: pledgedSrcSize = %u", (U32)pledgedSrcSize); if (pledgedSrcSize==0) pledgedSrcSize = ZSTD_CONTENTSIZE_UNKNOWN; params.fParams.contentSizeFlag = 1; - params.cParams = ZSTD_getCParamsFromCCtxParams(¶ms, pledgedSrcSize, 0); return ZSTD_resetCStream_internal(zcs, NULL, 0, ZSTD_dct_auto, zcs->cdict, params, pledgedSrcSize); } @@ -3376,6 +3377,7 @@ size_t ZSTD_initCStream_internal(ZSTD_CStream* zcs, ZSTD_CCtx_params params, unsigned long long pledgedSrcSize) { DEBUGLOG(4, "ZSTD_initCStream_internal"); + params.cParams = ZSTD_getCParamsFromCCtxParams(¶ms, pledgedSrcSize, dictSize); assert(!ZSTD_isError(ZSTD_checkCParams(params.cParams))); assert(!((dict) && (cdict))); /* either dict or cdict, not both */ @@ -3442,25 +3444,21 @@ size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, (U32)pledgedSrcSize, params.fParams.contentSizeFlag); CHECK_F( ZSTD_checkCParams(params.cParams) ); if ((pledgedSrcSize==0) && (params.fParams.contentSizeFlag==0)) pledgedSrcSize = ZSTD_CONTENTSIZE_UNKNOWN; /* for compatibility with older programs relying on this behavior. Users should now specify ZSTD_CONTENTSIZE_UNKNOWN. This line will be removed in the future. */ - { ZSTD_CCtx_params const cctxParams = ZSTD_assignParamsToCCtxParams(zcs->requestedParams, params); - return ZSTD_initCStream_internal(zcs, dict, dictSize, NULL /*cdict*/, cctxParams, pledgedSrcSize); - } + zcs->requestedParams = ZSTD_assignParamsToCCtxParams(zcs->requestedParams, params); + return ZSTD_initCStream_internal(zcs, dict, dictSize, NULL /*cdict*/, zcs->requestedParams, pledgedSrcSize); } size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel) { - ZSTD_parameters const params = ZSTD_getParams(compressionLevel, 0, dictSize); - ZSTD_CCtx_params const cctxParams = - ZSTD_assignParamsToCCtxParams(zcs->requestedParams, params); - return ZSTD_initCStream_internal(zcs, dict, dictSize, NULL, cctxParams, ZSTD_CONTENTSIZE_UNKNOWN); + ZSTD_CCtxParams_init(&zcs->requestedParams, compressionLevel); + return ZSTD_initCStream_internal(zcs, dict, dictSize, NULL, zcs->requestedParams, ZSTD_CONTENTSIZE_UNKNOWN); } size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pss) { U64 const pledgedSrcSize = (pss==0) ? ZSTD_CONTENTSIZE_UNKNOWN : pss; /* temporary : 0 interpreted as "unknown" during transition period. Users willing to specify "unknown" **must** use ZSTD_CONTENTSIZE_UNKNOWN. `0` will be interpreted as "empty" in the future */ - ZSTD_parameters const params = ZSTD_getParams(compressionLevel, pledgedSrcSize, 0); - ZSTD_CCtx_params const cctxParams = ZSTD_assignParamsToCCtxParams(zcs->requestedParams, params); - return ZSTD_initCStream_internal(zcs, NULL, 0, NULL, cctxParams, pledgedSrcSize); + ZSTD_CCtxParams_init(&zcs->requestedParams, compressionLevel); + return ZSTD_initCStream_internal(zcs, NULL, 0, NULL, zcs->requestedParams, pledgedSrcSize); } size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel) diff --git a/tests/zstreamtest.c b/tests/zstreamtest.c index 22c49cb35..3d61394a7 100644 --- a/tests/zstreamtest.c +++ b/tests/zstreamtest.c @@ -969,6 +969,26 @@ static int basicUnitTests(U32 seed, double compressibility) } DISPLAYLEVEL(3, "OK \n"); + DISPLAYLEVEL(3, "test%3i : ZSTD_initCStream_srcSize sets requestedParams : ", testNb++); + { unsigned level; + CHECK_Z(ZSTD_initCStream_srcSize(zc, 11, ZSTD_CONTENTSIZE_UNKNOWN)); + CHECK_Z(ZSTD_CCtx_getParameter(zc, ZSTD_p_compressionLevel, &level)); + CHECK(level != 11, "Compression level does not match"); + ZSTD_resetCStream(zc, ZSTD_CONTENTSIZE_UNKNOWN); + CHECK_Z(ZSTD_CCtx_getParameter(zc, ZSTD_p_compressionLevel, &level)); + CHECK(level != 11, "Compression level does not match"); + } + DISPLAYLEVEL(3, "OK \n"); + + DISPLAYLEVEL(3, "test%3i : ZSTD_initCStream_advanced sets requestedParams : ", testNb++); + { ZSTD_parameters const params = ZSTD_getParams(9, 0, 0); + CHECK_Z(ZSTD_initCStream_advanced(zc, NULL, 0, params, ZSTD_CONTENTSIZE_UNKNOWN)); + CHECK(badParameters(zc, params), "Compression parameters do not match"); + ZSTD_resetCStream(zc, ZSTD_CONTENTSIZE_UNKNOWN); + CHECK(badParameters(zc, params), "Compression parameters do not match"); + } + DISPLAYLEVEL(3, "OK \n"); + /* Overlen overwriting window data bug */ DISPLAYLEVEL(3, "test%3i : wildcopy doesn't overwrite potential match data : ", testNb++); { /* This test has a window size of 1024 bytes and consists of 3 blocks: From a23a3b95f9c00ecf52216bd7fe768e41eac4e269 Mon Sep 17 00:00:00 2001 From: Jennifer Liu Date: Fri, 13 Jul 2018 16:05:14 -0700 Subject: [PATCH 046/372] Add random dictionary builder --- contrib/randomDictBuilder/Makefile | 48 +++ contrib/randomDictBuilder/README.md | 13 + contrib/randomDictBuilder/main.c | 125 ++++++++ contrib/randomDictBuilder/random.c | 455 ++++++++++++++++++++++++++++ contrib/randomDictBuilder/random.h | 53 ++++ contrib/randomDictBuilder/test.sh | 14 + 6 files changed, 708 insertions(+) create mode 100644 contrib/randomDictBuilder/Makefile create mode 100644 contrib/randomDictBuilder/README.md create mode 100644 contrib/randomDictBuilder/main.c create mode 100644 contrib/randomDictBuilder/random.c create mode 100644 contrib/randomDictBuilder/random.h create mode 100644 contrib/randomDictBuilder/test.sh diff --git a/contrib/randomDictBuilder/Makefile b/contrib/randomDictBuilder/Makefile new file mode 100644 index 000000000..a2aade23a --- /dev/null +++ b/contrib/randomDictBuilder/Makefile @@ -0,0 +1,48 @@ +PROGRAM_FILES := ../../programs/fileio.c + +TEST_INPUT := ../../lib +TEST_OUTPUT := randomDict +ARG := + +all: main testrun test clean + +run: main rand clean + +.PHONY: rand +rand: + echo "Building a random dictionary with given arguments" + ./main $(ARG) + + +main: random.o main.o libzstd.a + gcc random.o main.o libzstd.a -o main + +main.o: main.c + gcc -c main.c -I ../../programs -I ../../lib/common -I ../../lib -I ../../lib/dictBuilder -I random.h + +random.o: $(PROGRAM_FILES) random.c + gcc -c $(PROGRAM_FILES) -I ../../programs -I ../../lib/common -I random.h random.c + +libzstd.a: + $(MAKE) -C ../../lib libzstd.a + mv ../../lib/libzstd.a . + +.PHONY: testrun +testrun: main + echo "Run with $(TEST_INPUT) and $(TEST_OUTPUT) " + ./main in=$(TEST_INPUT) out=$(TEST_OUTPUT) + zstd -be3 -D $(TEST_OUTPUT) -r $(TEST_INPUT) -q + rm -f $(TEST_OUTPUT) + +.PHONY: test +test: test.sh + sh test.sh + echo "Finish running test.sh" + +.PHONY: clean +clean: + rm -f libzstd.a main + rm -f ../../lib/*/*.o + rm -f ../../programs/*.o + rm -f *.o + echo "Cleaning is completed" diff --git a/contrib/randomDictBuilder/README.md b/contrib/randomDictBuilder/README.md new file mode 100644 index 000000000..cadffdf27 --- /dev/null +++ b/contrib/randomDictBuilder/README.md @@ -0,0 +1,13 @@ +Random Dictionary Builder + +### Permitted Arguments: +Input Files (in=fileName): files used to build dictionary, can include multiple files, each following "in=", required +Output Dictionary (out=dictName): if not provided, default to defaultDict +Dictionary ID (dictID=#): positive number, if not provided, default to 0 +Maximum Dictionary Size (maxdict=#): positive number, in bytes, if not provided, default to 110KB +Size of Randomly Selected Segment (k=#): positive number, in bytes, if not provided, default to 200 +Compression Level (c=#): positive number, if not provided, default to 3 + +### Examples: +make run ARG="in=../../lib/dictBuilder out=dict100 dictID=520" +make run ARG="in=../../lib/dictBuilder in=../../lib/compress" diff --git a/contrib/randomDictBuilder/main.c b/contrib/randomDictBuilder/main.c new file mode 100644 index 000000000..15eb5c440 --- /dev/null +++ b/contrib/randomDictBuilder/main.c @@ -0,0 +1,125 @@ +#include /* fprintf */ +#include /* malloc, free, qsort */ +#include /* strcmp, strlen */ +#include /* errno */ +#include +#include "fileio.h" /* stdinmark, stdoutmark, ZSTD_EXTENSION */ +#include "random.h" +#include "util.h" + +#define DISPLAY(...) fprintf(stderr, __VA_ARGS__) +#define DISPLAYLEVEL(l, ...) if (displayLevel>=l) { DISPLAY(__VA_ARGS__); } + +static const unsigned g_defaultMaxDictSize = 110 KB; +#define DEFAULT_CLEVEL 3 +#define DEFAULT_INPUTFILE "" +#define DEFAULT_k 200 +#define DEFAULT_OUTPUTFILE "defaultDict" +#define DEFAULT_DICTID 0 + + +static unsigned readU32FromChar(const char** stringPtr) +{ + const char errorMsg[] = "error: numeric value too large"; + unsigned result = 0; + while ((**stringPtr >='0') && (**stringPtr <='9')) { + unsigned const max = (((unsigned)(-1)) / 10) - 1; + if (result > max) exit(1); + result *= 10, result += **stringPtr - '0', (*stringPtr)++ ; + } + if ((**stringPtr=='K') || (**stringPtr=='M')) { + unsigned const maxK = ((unsigned)(-1)) >> 10; + if (result > maxK) exit(1); + result <<= 10; + if (**stringPtr=='M') { + if (result > maxK) exit(1); + result <<= 10; + } + (*stringPtr)++; /* skip `K` or `M` */ + if (**stringPtr=='i') (*stringPtr)++; + if (**stringPtr=='B') (*stringPtr)++; + } + return result; +} + + +/** longCommandWArg() : + * check if *stringPtr is the same as longCommand. + * If yes, @return 1 and advances *stringPtr to the position which immediately follows longCommand. + * @return 0 and doesn't modify *stringPtr otherwise. + */ +static unsigned longCommandWArg(const char** stringPtr, const char* longCommand) +{ + size_t const comSize = strlen(longCommand); + int const result = !strncmp(*stringPtr, longCommand, comSize); + if (result) *stringPtr += comSize; + return result; +} + + +int main(int argCount, const char* argv[]) +{ + int displayLevel = 2; + const char* programName = argv[0]; + int operationResult = 0; + + unsigned cLevel = DEFAULT_CLEVEL; + char* inputFile = DEFAULT_INPUTFILE; + unsigned k = DEFAULT_k; + char* outputFile = DEFAULT_OUTPUTFILE; + unsigned dictID = DEFAULT_DICTID; + unsigned maxDictSize = g_defaultMaxDictSize; + + const char** filenameTable = (const char**)malloc(argCount * sizeof(const char*)); + unsigned filenameIdx = 0; + + for (int i = 1; i < argCount; i++) { + const char* argument = argv[i]; + if (longCommandWArg(&argument, "k=")) { k = readU32FromChar(&argument); continue; } + if (longCommandWArg(&argument, "c=")) { cLevel = readU32FromChar(&argument); continue; } + if (longCommandWArg(&argument, "dictID=")) { dictID = readU32FromChar(&argument); continue; } + if (longCommandWArg(&argument, "maxdict=")) { maxDictSize = readU32FromChar(&argument); continue; } + if (longCommandWArg(&argument, "in=")) { + inputFile = malloc(strlen(argument) + 1); + strcpy(inputFile, argument); + filenameTable[filenameIdx] = inputFile; + filenameIdx++; + continue; + } + if (longCommandWArg(&argument, "out=")) { + outputFile = malloc(strlen(argument) + 1); + strcpy(outputFile, argument); + continue; + } + DISPLAYLEVEL(1, "Incorrect parameters\n"); + operationResult = 1; + return operationResult; + } + + + char* fileNamesBuf = NULL; + unsigned fileNamesNb = filenameIdx; + int followLinks = 0; + const char** extendedFileList = NULL; + extendedFileList = UTIL_createFileList(filenameTable, filenameIdx, &fileNamesBuf, &fileNamesNb, followLinks); + if (extendedFileList) { + unsigned u; + for (u=0; u /* fprintf */ +#include /* malloc, free, qsort */ +#include /* memset */ +#include /* clock */ +#include "zstd_internal.h" /* includes zstd.h */ +#ifndef ZDICT_STATIC_LINKING_ONLY +#define ZDICT_STATIC_LINKING_ONLY +#endif +#include "random.h" +#include "platform.h" /* Large Files support */ +#include "util.h" /* UTIL_getFileSize, UTIL_getTotalFileSize */ + +/*-************************************* +* Constants +***************************************/ +#define SAMPLESIZE_MAX (128 KB) +#define RANDOM_MAX_SAMPLES_SIZE (sizeof(size_t) == 8 ? ((U32)-1) : ((U32)1 GB)) +#define RANDOM_MEMMULT 9 +static const size_t g_maxMemory = (sizeof(size_t) == 4) ? (2 GB - 64 MB) : ((size_t)(512 MB) << sizeof(size_t)); + +#define NOISELENGTH 32 +#define DEFAULT_K 200 + +/*-************************************* +* Console display +***************************************/ +#define DISPLAY(...) fprintf(stderr, __VA_ARGS__) +#define DISPLAYLEVEL(l, ...) if (displayLevel>=l) { DISPLAY(__VA_ARGS__); } + +static const U64 g_refreshRate = SEC_TO_MICRO / 6; +static UTIL_time_t g_displayClock = UTIL_TIME_INITIALIZER; + +#define DISPLAYUPDATE(l, ...) { if (displayLevel>=l) { \ + if ((UTIL_clockSpanMicro(g_displayClock) > g_refreshRate) || (displayLevel>=4)) \ + { g_displayClock = UTIL_getTime(); DISPLAY(__VA_ARGS__); \ + if (displayLevel>=4) fflush(stderr); } } } + + +/*-************************************* +* Exceptions +***************************************/ +#ifndef DEBUG +# define DEBUG 0 +#endif +#define DEBUGOUTPUT(...) if (DEBUG) DISPLAY(__VA_ARGS__); +#define EXM_THROW(error, ...) \ +{ \ + DEBUGOUTPUT("Error defined at %s, line %i : \n", __FILE__, __LINE__); \ + DISPLAY("Error %i : ", error); \ + DISPLAY(__VA_ARGS__); \ + DISPLAY("\n"); \ + exit(error); \ +} + + +/* ******************************************************** +* File related operations +**********************************************************/ +/** loadFiles() : + * load samples from files listed in fileNamesTable into buffer. + * works even if buffer is too small to load all samples. + * Also provides the size of each sample into sampleSizes table + * which must be sized correctly, using DiB_fileStats(). + * @return : nb of samples effectively loaded into `buffer` + * *bufferSizePtr is modified, it provides the amount data loaded within buffer. + * sampleSizes is filled with the size of each sample. + */ +static unsigned loadFiles(void* buffer, size_t* bufferSizePtr, + size_t* sampleSizes, unsigned sstSize, + const char** fileNamesTable, unsigned nbFiles, size_t targetChunkSize, + unsigned displayLevel) +{ + char* const buff = (char*)buffer; + size_t pos = 0; + unsigned nbLoadedChunks = 0, fileIndex; + + for (fileIndex=0; fileIndex *bufferSizePtr-pos) break; + { size_t const readSize = fread(buff+pos, 1, toLoad, f); + if (readSize != toLoad) EXM_THROW(11, "Pb reading %s", fileName); + pos += readSize; + sampleSizes[nbLoadedChunks++] = toLoad; + remainingToLoad -= targetChunkSize; + if (nbLoadedChunks == sstSize) { /* no more space left in sampleSizes table */ + fileIndex = nbFiles; /* stop there */ + break; + } + if (toLoad < targetChunkSize) { + fseek(f, (long)(targetChunkSize - toLoad), SEEK_CUR); + } } } + fclose(f); + } + DISPLAYLEVEL(2, "\r%79s\r", ""); + *bufferSizePtr = pos; + DISPLAYLEVEL(4, "loaded : %u KB \n", (U32)(pos >> 10)) + return nbLoadedChunks; +} + + + +#define rotl32(x,r) ((x << r) | (x >> (32 - r))) +static U32 getRand(U32* src) +{ + static const U32 prime1 = 2654435761U; + static const U32 prime2 = 2246822519U; + U32 rand32 = *src; + rand32 *= prime1; + rand32 ^= prime2; + rand32 = rotl32(rand32, 13); + *src = rand32; + return rand32 >> 5; +} + + +/* shuffle() : + * shuffle a table of file names in a semi-random way + * It improves dictionary quality by reducing "locality" impact, so if sample set is very large, + * it will load random elements from it, instead of just the first ones. */ +static void shuffle(const char** fileNamesTable, unsigned nbFiles) { + U32 seed = 0xFD2FB528; + unsigned i; + for (i = nbFiles - 1; i > 0; --i) { + unsigned const j = getRand(&seed) % (i + 1); + const char* const tmp = fileNamesTable[j]; + fileNamesTable[j] = fileNamesTable[i]; + fileNamesTable[i] = tmp; + } +} + + + +/*-******************************************************** +* Dictionary training functions +**********************************************************/ +static size_t findMaxMem(unsigned long long requiredMem) +{ + size_t const step = 8 MB; + void* testmem = NULL; + + requiredMem = (((requiredMem >> 23) + 1) << 23); + requiredMem += step; + if (requiredMem > g_maxMemory) requiredMem = g_maxMemory; + + while (!testmem) { + testmem = malloc((size_t)requiredMem); + requiredMem -= step; + } + + free(testmem); + return (size_t)requiredMem; +} + +static void saveDict(const char* dictFileName, + const void* buff, size_t buffSize) +{ + FILE* const f = fopen(dictFileName, "wb"); + if (f==NULL) EXM_THROW(3, "cannot open %s ", dictFileName); + + { size_t const n = fwrite(buff, 1, buffSize, f); + if (n!=buffSize) EXM_THROW(4, "%s : write error", dictFileName) } + + { size_t const n = (size_t)fclose(f); + if (n!=0) EXM_THROW(5, "%s : flush error", dictFileName) } +} + +/*! getFileStats() : + * Given a list of files, and a chunkSize (0 == no chunk, whole files) + * provides the amount of data to be loaded and the resulting nb of samples. + * This is useful primarily for allocation purpose => sample buffer, and sample sizes table. + */ +static fileStats getFileStats(const char** fileNamesTable, unsigned nbFiles, size_t chunkSize, unsigned displayLevel) +{ + fileStats fs; + unsigned n; + memset(&fs, 0, sizeof(fs)); + for (n=0; n 2*SAMPLESIZE_MAX); + fs.nbSamples += nbSamples; + } + DISPLAYLEVEL(4, "Preparing to load : %u KB \n", (U32)(fs.totalSizeToLoad >> 10)); + return fs; +} + + + + + +/* ******************************************************** +* Random Dictionary Builder +**********************************************************/ +/** + * Returns the sum of the sample sizes. + */ +static size_t RANDOM_sum(const size_t *samplesSizes, unsigned nbSamples) { + size_t sum = 0; + unsigned i; + for (i = 0; i < nbSamples; ++i) { + sum += samplesSizes[i]; + } + return sum; +} + + +/** + * Selects a random segment from totalSamplesSize - k + 1 possible segments + */ +static RANDOM_segment_t RANDOM_selectSegment(const RANDOM_ctx_t *ctx, + ZDICT_random_params_t parameters) { + const U32 k = parameters.k; + RANDOM_segment_t segment; + unsigned index; + + /* Seed random number generator */ + srand((unsigned)time(NULL)); + /* Randomly generate a number from 0 to sampleSizes - k */ + index = rand()%(ctx->totalSamplesSize - k + 1); + + /* inclusive */ + segment.begin = index; + segment.end = index + k - 1; + + return segment; +} + + +/** + * Check the validity of the parameters. + * Returns non-zero if the parameters are valid and 0 otherwise. + */ +static int RANDOM_checkParameters(ZDICT_random_params_t parameters, size_t maxDictSize) { + /* k is a required parameter */ + if (parameters.k == 0) { + return 0; + } + /* k <= maxDictSize */ + if (parameters.k > maxDictSize) { + return 0; + } + return 1; +} + + +/** + * Clean up a context initialized with `RANDOM_ctx_init()`. + */ +static void RANDOM_ctx_destroy(RANDOM_ctx_t *ctx) { + if (!ctx) { + return; + } + if (ctx->offsets) { + free(ctx->offsets); + ctx->offsets = NULL; + } +} + + +/** + * Prepare a context for dictionary building. + * Returns 1 on success or zero on error. + * The context must be destroyed with `RANDOM_ctx_destroy()`. + */ +static int RANDOM_ctx_init(RANDOM_ctx_t *ctx, const void *samplesBuffer, + const size_t *samplesSizes, unsigned nbSamples) { + const BYTE *const samples = (const BYTE *)samplesBuffer; + const size_t totalSamplesSize = RANDOM_sum(samplesSizes, nbSamples); + const int displayLevel = 2; + /* Checks */ + if (totalSamplesSize >= (size_t)RANDOM_MAX_SAMPLES_SIZE) { + DISPLAYLEVEL(1, "Total samples size is too large (%u MB), maximum size is %u MB\n", + (U32)(totalSamplesSize>>20), (RANDOM_MAX_SAMPLES_SIZE >> 20)); + return 0; + } + memset(ctx, 0, sizeof(*ctx)); + DISPLAYLEVEL(1, "Building dictionary from %u samples of total size %u\n", nbSamples, + (U32)totalSamplesSize); + ctx->samples = samples; + ctx->samplesSizes = samplesSizes; + ctx->nbSamples = nbSamples; + ctx->offsets = (size_t *)malloc((nbSamples + 1) * sizeof(size_t)); + ctx->totalSamplesSize = (U32)totalSamplesSize; + if (!ctx->offsets) { + DISPLAYLEVEL(1, "Failed to allocate buffer for offsets\n"); + RANDOM_ctx_destroy(ctx); + return 0; + } + { + U32 i; + ctx->offsets[0] = 0; + for (i = 1; i <= nbSamples; ++i) { + ctx->offsets[i] = ctx->offsets[i - 1] + samplesSizes[i - 1]; + } + } + return 1; +} + + +/** + * Given the prepared context build the dictionary. + */ +static size_t RANDOM_buildDictionary(const RANDOM_ctx_t *ctx, void *dictBuffer, + size_t dictBufferCapacity, + ZDICT_random_params_t parameters) { + BYTE *const dict = (BYTE *)dictBuffer; + size_t tail = dictBufferCapacity; + const int displayLevel = parameters.zParams.notificationLevel; + while (tail > 0) { + + /* Select a segment */ + RANDOM_segment_t segment = RANDOM_selectSegment(ctx, parameters); + + size_t segmentSize; + segmentSize = MIN(segment.end - segment.begin + 1, tail); + + tail -= segmentSize; + memcpy(dict + tail, ctx->samples + segment.begin, segmentSize); + DISPLAYUPDATE( + 2, "\r%u%% ", + (U32)(((dictBufferCapacity - tail) * 100) / dictBufferCapacity)); + } + + return tail; +} + +/*! ZDICT_trainFromBuffer_random(): + * Train a dictionary from an array of samples using the RANDOM algorithm. + * Samples must be stored concatenated in a single flat buffer `samplesBuffer`, + * supplied with an array of sizes `samplesSizes`, providing the size of each sample, in order. + * The resulting dictionary will be saved into `dictBuffer`. + * @return: size of dictionary stored into `dictBuffer` (<= `dictBufferCapacity`) + * or an error code, which can be tested with ZDICT_isError(). + */ +ZDICTLIB_API size_t ZDICT_trainFromBuffer_random( + void *dictBuffer, size_t dictBufferCapacity, + const void *samplesBuffer, const size_t *samplesSizes, unsigned nbSamples, + ZDICT_random_params_t parameters) { + const int displayLevel = parameters.zParams.notificationLevel; + BYTE* const dict = (BYTE*)dictBuffer; + RANDOM_ctx_t ctx; + /* Checks */ + if (!RANDOM_checkParameters(parameters, dictBufferCapacity)) { + DISPLAYLEVEL(1, "k is incorrect\n"); + return ERROR(GENERIC); + } + if (nbSamples == 0) { + DISPLAYLEVEL(1, "Random must have at least one input file\n"); + return ERROR(GENERIC); + } + if (dictBufferCapacity < ZDICT_DICTSIZE_MIN) { + DISPLAYLEVEL(1, "dictBufferCapacity must be at least %u\n", + ZDICT_DICTSIZE_MIN); + return ERROR(dstSize_tooSmall); + } + + if (!RANDOM_ctx_init(&ctx, samplesBuffer, samplesSizes, nbSamples)) { + return ERROR(GENERIC); + } + DISPLAYLEVEL(2, "Building dictionary\n"); + { + const size_t tail = RANDOM_buildDictionary(&ctx, dictBuffer, dictBufferCapacity, parameters); + const size_t dictSize = ZDICT_finalizeDictionary( + dict, dictBufferCapacity, dict + tail, dictBufferCapacity - tail, + samplesBuffer, samplesSizes, nbSamples, parameters.zParams); + if (!ZSTD_isError(dictSize)) { + DISPLAYLEVEL(2, "Constructed dictionary of size %u\n", + (U32)dictSize); + } + RANDOM_ctx_destroy(&ctx); + return dictSize; + } +} + + +int RANDOM_trainFromFiles(const char* dictFileName, unsigned maxDictSize, + const char** fileNamesTable, unsigned nbFiles, + size_t chunkSize, ZDICT_random_params_t *params){ + unsigned const displayLevel = params->zParams.notificationLevel; + void* const dictBuffer = malloc(maxDictSize); + fileStats const fs = getFileStats(fileNamesTable, nbFiles, chunkSize, displayLevel); + size_t* const sampleSizes = (size_t*)malloc(fs.nbSamples * sizeof(size_t)); + size_t const memMult = RANDOM_MEMMULT; + size_t const maxMem = findMaxMem(fs.totalSizeToLoad * memMult) / memMult; + size_t loadedSize = (size_t) MIN ((unsigned long long)maxMem, fs.totalSizeToLoad); + void* const srcBuffer = malloc(loadedSize+NOISELENGTH); + int result = 0; + + /* Checks */ + if ((!sampleSizes) || (!srcBuffer) || (!dictBuffer)) + EXM_THROW(12, "not enough memory for DiB_trainFiles"); /* should not happen */ + if (fs.oneSampleTooLarge) { + DISPLAYLEVEL(2, "! Warning : some sample(s) are very large \n"); + DISPLAYLEVEL(2, "! Note that dictionary is only useful for small samples. \n"); + DISPLAYLEVEL(2, "! As a consequence, only the first %u bytes of each sample are loaded \n", SAMPLESIZE_MAX); + } + if (fs.nbSamples < 5) { + DISPLAYLEVEL(2, "! Warning : nb of samples too low for proper processing ! \n"); + DISPLAYLEVEL(2, "! Please provide _one file per sample_. \n"); + DISPLAYLEVEL(2, "! Alternatively, split files into fixed-size blocks representative of samples, with -B# \n"); + EXM_THROW(14, "nb of samples too low"); /* we now clearly forbid this case */ + } + if (fs.totalSizeToLoad < (unsigned long long)(8 * maxDictSize)) { + DISPLAYLEVEL(2, "! Warning : data size of samples too small for target dictionary size \n"); + DISPLAYLEVEL(2, "! Samples should be about 100x larger than target dictionary size \n"); + } + + /* init */ + if (loadedSize < fs.totalSizeToLoad) + DISPLAYLEVEL(1, "Not enough memory; training on %u MB only...\n", (unsigned)(loadedSize >> 20)); + + /* Load input buffer */ + DISPLAYLEVEL(3, "Shuffling input files\n"); + shuffle(fileNamesTable, nbFiles); + nbFiles = loadFiles(srcBuffer, &loadedSize, sampleSizes, fs.nbSamples, fileNamesTable, nbFiles, chunkSize, displayLevel); + + { size_t dictSize; + dictSize = ZDICT_trainFromBuffer_random(dictBuffer, maxDictSize, srcBuffer, + sampleSizes, fs.nbSamples, *params); + DISPLAYLEVEL(2, "k=%u\n", params->k); + if (ZDICT_isError(dictSize)) { + DISPLAYLEVEL(1, "dictionary training failed : %s \n", ZDICT_getErrorName(dictSize)); /* should not happen */ + result = 1; + goto _cleanup; + } + /* save dict */ + DISPLAYLEVEL(2, "Save dictionary of size %u into file %s \n", (U32)dictSize, dictFileName); + saveDict(dictFileName, dictBuffer, dictSize); + } + + /* clean up */ +_cleanup: + free(srcBuffer); + free(sampleSizes); + free(dictBuffer); + return result; +} diff --git a/contrib/randomDictBuilder/random.h b/contrib/randomDictBuilder/random.h new file mode 100644 index 000000000..058796414 --- /dev/null +++ b/contrib/randomDictBuilder/random.h @@ -0,0 +1,53 @@ +#include /* fprintf */ +#include /* malloc, free, qsort */ +#include /* memset */ +#include /* clock */ +#include "zstd_internal.h" /* includes zstd.h */ +#ifndef ZDICT_STATIC_LINKING_ONLY +#define ZDICT_STATIC_LINKING_ONLY +#endif +#include "zdict.h" + + +/************************************** +* Context +***************************************/ +typedef struct { + const BYTE *samples; + size_t *offsets; + const size_t *samplesSizes; + size_t nbSamples; + U32 totalSamplesSize; +} RANDOM_ctx_t; + +/** + * A segment is an inclusive range in the source. + */ +typedef struct { + U32 begin; + U32 end; +} RANDOM_segment_t; + + +typedef struct { + unsigned k; /* Segment size : constraint: 0 < k : Reasonable range [16, 2048+]; Default to 200 */ + ZDICT_params_t zParams; +} ZDICT_random_params_t; + + +typedef struct { + U64 totalSizeToLoad; + unsigned oneSampleTooLarge; + unsigned nbSamples; +} fileStats; + + +ZDICTLIB_API size_t ZDICT_trainFromBuffer_random( + void *dictBuffer, size_t dictBufferCapacity, + const void *samplesBuffer, const size_t *samplesSizes, unsigned nbSamples, + ZDICT_random_params_t parameters); + + +int RANDOM_trainFromFiles(const char* dictFileName, unsigned maxDictSize, + const char** fileNamesTable, unsigned nbFiles, + size_t chunkSize, ZDICT_random_params_t *params); diff --git a/contrib/randomDictBuilder/test.sh b/contrib/randomDictBuilder/test.sh new file mode 100644 index 000000000..552650ee4 --- /dev/null +++ b/contrib/randomDictBuilder/test.sh @@ -0,0 +1,14 @@ +echo "Building random dictionary with c=5 in=../../lib/common k=200 out=dict1" +./main c=5 in=../../lib/common k=200 out=dict1 +zstd -be3 -D dict1 -r ../../lib/common -q +echo "Building random dictionary with c=9 in=../../lib/common k=500 out=dict2 dictID=100 maxdict=140000" +./main c=9 in=../../lib/common k=500 out=dict2 dictID=100 maxdict=140000 +zstd -be3 -D dict2 -r ../../lib/common -q +echo "Building random dictionary with 2 sample sources" +./main in=../../lib/common in=../../lib/compress out=dict3 +zstd -be3 -D dict3 -r ../../lib/common -q +echo "Removing dict1 dict2 dict3" +rm -f dict1 dict2 dict3 + +echo "Testing with invalid parameters, should fail" +! ./main r=10 From 31731df4dab0df7b465de2de5641b2e3416c9086 Mon Sep 17 00:00:00 2001 From: Jennifer Liu Date: Fri, 13 Jul 2018 17:38:53 -0700 Subject: [PATCH 047/372] Remove clevel and update documentation --- contrib/randomDictBuilder/README.md | 15 ++++++++++----- contrib/randomDictBuilder/main.c | 11 ++++++++--- contrib/randomDictBuilder/test.sh | 8 ++++---- 3 files changed, 22 insertions(+), 12 deletions(-) diff --git a/contrib/randomDictBuilder/README.md b/contrib/randomDictBuilder/README.md index cadffdf27..de2c7ff6c 100644 --- a/contrib/randomDictBuilder/README.md +++ b/contrib/randomDictBuilder/README.md @@ -1,12 +1,17 @@ Random Dictionary Builder ### Permitted Arguments: -Input Files (in=fileName): files used to build dictionary, can include multiple files, each following "in=", required +Input File/Directory (in=fileName): required; file/directory used to build dictionary; if directory, will operate recursively for files inside directory; can include multiple files/directories, each following "in=" Output Dictionary (out=dictName): if not provided, default to defaultDict -Dictionary ID (dictID=#): positive number, if not provided, default to 0 -Maximum Dictionary Size (maxdict=#): positive number, in bytes, if not provided, default to 110KB -Size of Randomly Selected Segment (k=#): positive number, in bytes, if not provided, default to 200 -Compression Level (c=#): positive number, if not provided, default to 3 +Dictionary ID (dictID=#): nonnegative number; if not provided, default to 0 +Maximum Dictionary Size (maxdict=#): positive number; in bytes, if not provided, default to 110KB +Size of Randomly Selected Segment (k=#): positive number; in bytes; if not provided, default to 200 +Compression Level (c=#): positive number; if not provided, default to 3 + + +###Usage: +To build a random dictionary with the provided arguments: make run ARG= followed by arguments + ### Examples: make run ARG="in=../../lib/dictBuilder out=dict100 dictID=520" diff --git a/contrib/randomDictBuilder/main.c b/contrib/randomDictBuilder/main.c index 15eb5c440..cf0b94769 100644 --- a/contrib/randomDictBuilder/main.c +++ b/contrib/randomDictBuilder/main.c @@ -63,7 +63,7 @@ int main(int argCount, const char* argv[]) const char* programName = argv[0]; int operationResult = 0; - unsigned cLevel = DEFAULT_CLEVEL; + /* Initialize parameters with default value */ char* inputFile = DEFAULT_INPUTFILE; unsigned k = DEFAULT_k; char* outputFile = DEFAULT_OUTPUTFILE; @@ -76,10 +76,10 @@ int main(int argCount, const char* argv[]) for (int i = 1; i < argCount; i++) { const char* argument = argv[i]; if (longCommandWArg(&argument, "k=")) { k = readU32FromChar(&argument); continue; } - if (longCommandWArg(&argument, "c=")) { cLevel = readU32FromChar(&argument); continue; } if (longCommandWArg(&argument, "dictID=")) { dictID = readU32FromChar(&argument); continue; } if (longCommandWArg(&argument, "maxdict=")) { maxDictSize = readU32FromChar(&argument); continue; } if (longCommandWArg(&argument, "in=")) { + /* Allow multiple input files */ inputFile = malloc(strlen(argument) + 1); strcpy(inputFile, argument); filenameTable[filenameIdx] = inputFile; @@ -96,6 +96,11 @@ int main(int argCount, const char* argv[]) return operationResult; } + if (maxDictSize == 0) { + DISPLAYLEVEL(1, "maxDictSize should not be 0.\n"); + operationResult = 1; + return operationResult; + } char* fileNamesBuf = NULL; unsigned fileNamesNb = filenameIdx; @@ -114,7 +119,7 @@ int main(int argCount, const char* argv[]) ZDICT_random_params_t params; ZDICT_params_t zParams; - zParams.compressionLevel = cLevel; + zParams.compressionLevel = DEFAULT_CLEVEL; zParams.notificationLevel = displayLevel; zParams.dictID = dictID; params.zParams = zParams; diff --git a/contrib/randomDictBuilder/test.sh b/contrib/randomDictBuilder/test.sh index 552650ee4..497820f88 100644 --- a/contrib/randomDictBuilder/test.sh +++ b/contrib/randomDictBuilder/test.sh @@ -1,8 +1,8 @@ -echo "Building random dictionary with c=5 in=../../lib/common k=200 out=dict1" -./main c=5 in=../../lib/common k=200 out=dict1 +echo "Building random dictionary with in=../../lib/common k=200 out=dict1" +./main in=../../lib/common k=200 out=dict1 zstd -be3 -D dict1 -r ../../lib/common -q -echo "Building random dictionary with c=9 in=../../lib/common k=500 out=dict2 dictID=100 maxdict=140000" -./main c=9 in=../../lib/common k=500 out=dict2 dictID=100 maxdict=140000 +echo "Building random dictionary with in=../../lib/common k=500 out=dict2 dictID=100 maxdict=140000" +./main in=../../lib/common k=500 out=dict2 dictID=100 maxdict=140000 zstd -be3 -D dict2 -r ../../lib/common -q echo "Building random dictionary with 2 sample sources" ./main in=../../lib/common in=../../lib/compress out=dict3 From 0e5fbc10facdce2def08e4f4ecb67d255694df3a Mon Sep 17 00:00:00 2001 From: Jennifer Liu Date: Fri, 13 Jul 2018 17:41:09 -0700 Subject: [PATCH 048/372] Update README --- contrib/randomDictBuilder/README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/contrib/randomDictBuilder/README.md b/contrib/randomDictBuilder/README.md index de2c7ff6c..09f1e8088 100644 --- a/contrib/randomDictBuilder/README.md +++ b/contrib/randomDictBuilder/README.md @@ -6,7 +6,6 @@ Output Dictionary (out=dictName): if not provided, default to defaultDict Dictionary ID (dictID=#): nonnegative number; if not provided, default to 0 Maximum Dictionary Size (maxdict=#): positive number; in bytes, if not provided, default to 110KB Size of Randomly Selected Segment (k=#): positive number; in bytes; if not provided, default to 200 -Compression Level (c=#): positive number; if not provided, default to 3 ###Usage: From 1a61bdb9c08c9b4edc1891c78514510d67906789 Mon Sep 17 00:00:00 2001 From: Codecat Date: Sat, 14 Jul 2018 12:27:42 +0200 Subject: [PATCH 049/372] Update zstd.lua --- contrib/premake/zstd.lua | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/contrib/premake/zstd.lua b/contrib/premake/zstd.lua index 4759dff76..0f7fc558b 100644 --- a/contrib/premake/zstd.lua +++ b/contrib/premake/zstd.lua @@ -57,11 +57,12 @@ function project_zstd(dir, compression, decompression, deprecated, dictbuilder, } end - if legacy < 8 then - files { - dir .. 'legacy/zstd_v0' .. (legacy - 7) .. '.*' - } - else + if legacy ~= 0 then + if legacy >= 8 then + files { + dir .. 'legacy/zstd_v0' .. (legacy - 7) .. '.*' + } + end includedirs { dir .. 'legacy' } From 044cd81ce6cf4ef6c15bec933166a66a4528bc29 Mon Sep 17 00:00:00 2001 From: Codecat Date: Sat, 14 Jul 2018 12:34:03 +0200 Subject: [PATCH 050/372] Fix wrong conditions --- contrib/premake/zstd.lua | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/contrib/premake/zstd.lua b/contrib/premake/zstd.lua index 0f7fc558b..df1ace3ee 100644 --- a/contrib/premake/zstd.lua +++ b/contrib/premake/zstd.lua @@ -9,12 +9,12 @@ function project_zstd(dir, compression, decompression, deprecated, dictbuilder, if legacy == nil then legacy = 0 end - if compression then + if not compression then dictbuilder = false deprecated = false end - if decompression then + if not decompression then legacy = 0 deprecated = false end From 58b82194755b52ad80b6e7da5aeae8e383f8bb90 Mon Sep 17 00:00:00 2001 From: Nick Terrell Date: Mon, 9 Jul 2018 18:24:07 -0700 Subject: [PATCH 051/372] zstdcli: Allow -o before --train Only set the default value if `outFileName` is unset. Fixes #1227. --- programs/zstdcli.c | 14 ++++++++------ tests/playTests.sh | 24 +++++++++++++++++++++--- 2 files changed, 29 insertions(+), 9 deletions(-) diff --git a/programs/zstdcli.c b/programs/zstdcli.c index e0d7807f9..36ba21156 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -502,7 +502,7 @@ int main(int argCount, const char* argv[]) if (!strcmp(argument, "--sparse")) { FIO_setSparseWrite(2); continue; } if (!strcmp(argument, "--no-sparse")) { FIO_setSparseWrite(0); continue; } if (!strcmp(argument, "--test")) { operation=zom_test; continue; } - if (!strcmp(argument, "--train")) { operation=zom_train; outFileName=g_defaultDictName; continue; } + if (!strcmp(argument, "--train")) { operation=zom_train; if (outFileName==NULL) outFileName=g_defaultDictName; continue; } if (!strcmp(argument, "--maxdict")) { nextArgumentIsMaxDict=1; lastCommand=1; continue; } /* kept available for compatibility with old syntax ; will be removed one day */ if (!strcmp(argument, "--dictID")) { nextArgumentIsDictID=1; lastCommand=1; continue; } /* kept available for compatibility with old syntax ; will be removed one day */ if (!strcmp(argument, "--no-dictID")) { FIO_setDictIDFlag(0); continue; } @@ -526,7 +526,8 @@ int main(int argCount, const char* argv[]) #ifndef ZSTD_NODICT if (longCommandWArg(&argument, "--train-cover")) { operation = zom_train; - outFileName = g_defaultDictName; + if (outFileName == NULL) + outFileName = g_defaultDictName; cover = 1; /* Allow optional arguments following an = */ if (*argument == 0) { memset(&coverParams, 0, sizeof(coverParams)); } @@ -536,7 +537,8 @@ int main(int argCount, const char* argv[]) } if (longCommandWArg(&argument, "--train-legacy")) { operation = zom_train; - outFileName = g_defaultDictName; + if (outFileName == NULL) + outFileName = g_defaultDictName; cover = 0; /* Allow optional arguments following an = */ if (*argument == 0) { continue; } @@ -718,7 +720,7 @@ int main(int argCount, const char* argv[]) break; /* Select compressibility of synthetic sample */ - case 'P': + case 'P': { argument++; compressibility = (double)readU32FromChar(&argument) / 100; } @@ -841,7 +843,7 @@ int main(int argCount, const char* argv[]) if (cLevel > ZSTD_maxCLevel()) cLevel = ZSTD_maxCLevel(); if (cLevelLast > ZSTD_maxCLevel()) cLevelLast = ZSTD_maxCLevel(); if (cLevelLast < cLevel) cLevelLast = cLevel; - if (cLevelLast > cLevel) + if (cLevelLast > cLevel) DISPLAYLEVEL(2, "Benchmarking levels from %d to %d\n", cLevel, cLevelLast); if(filenameIdx) { if(separateFiles) { @@ -856,7 +858,7 @@ int main(int argCount, const char* argv[]) } else { for(; cLevel <= cLevelLast; cLevel++) { BMK_benchFilesAdvanced(filenameTable, filenameIdx, dictFileName, cLevel, &compressionParams, g_displayLevel, &adv); - } + } } } else { for(; cLevel <= cLevelLast; cLevel++) { diff --git a/tests/playTests.sh b/tests/playTests.sh index fb8b1d24f..0a1f96c02 100755 --- a/tests/playTests.sh +++ b/tests/playTests.sh @@ -404,7 +404,13 @@ $ECHO "Hello World" > tmp $ZSTD --train-legacy -q tmp && die "Dictionary training should fail : not enough input source" ./datagen -P0 -g10M > tmp $ZSTD --train-legacy -q tmp && die "Dictionary training should fail : source is pure noise" -rm tmp* +$ECHO "- Test -o before --train" +rm -f tmpDict dictionary +$ZSTD -o tmpDict --train *.c ../programs/*.c +test -f tmpDict +$ZSTD --train *.c ../programs/*.c +test -f dictionary +rm tmp* dictionary $ECHO "\n===> cover dictionary builder : advanced options " @@ -425,12 +431,18 @@ $ZSTD --train-cover=k=46,d=8 *.c ../programs/*.c --dictID=1 -o tmpDict1 cmp tmpDict tmpDict1 && die "dictionaries should have different ID !" $ECHO "- Create dictionary with size limit" $ZSTD --train-cover=steps=8 *.c ../programs/*.c -o tmpDict2 --maxdict=4K -rm tmp* $ECHO "- Compare size of dictionary from 90% training samples with 80% training samples" $ZSTD --train-cover=split=90 -r *.c ../programs/*.c $ZSTD --train-cover=split=80 -r *.c ../programs/*.c $ECHO "- Create dictionary using all samples for both training and testing" $ZSTD --train-cover=split=100 -r *.c ../programs/*.c +$ECHO "- Test -o before --train-cover" +rm -f tmpDict dictionary +$ZSTD -o tmpDict --train-cover *.c ../programs/*.c +test -f tmpDict +$ZSTD --train-cover *.c ../programs/*.c +test -f dictionary +rm tmp* dictionary $ECHO "\n===> legacy dictionary builder " @@ -450,7 +462,13 @@ $ZSTD --train-legacy -s5 *.c ../programs/*.c --dictID=1 -o tmpDict1 cmp tmpDict tmpDict1 && die "dictionaries should have different ID !" $ECHO "- Create dictionary with size limit" $ZSTD --train-legacy -s9 *.c ../programs/*.c -o tmpDict2 --maxdict=4K -rm tmp* +$ECHO "- Test -o before --train-legacy" +rm -f tmpDict dictionary +$ZSTD -o tmpDict --train-legacy *.c ../programs/*.c +test -f tmpDict +$ZSTD --train-legacy *.c ../programs/*.c +test -f dictionary +rm tmp* dictionary $ECHO "\n===> integrity tests " From b5806d33db813dfb2bac7cd3b97b5bcf09ee57b7 Mon Sep 17 00:00:00 2001 From: Jennifer Liu Date: Mon, 16 Jul 2018 16:03:04 -0700 Subject: [PATCH 052/372] Refactor RANDOM --- contrib/randomDictBuilder/Makefile | 12 +- contrib/randomDictBuilder/main.c | 297 ++++++++++++++++++++++++- contrib/randomDictBuilder/random.c | 343 ++--------------------------- contrib/randomDictBuilder/random.h | 23 -- 4 files changed, 314 insertions(+), 361 deletions(-) diff --git a/contrib/randomDictBuilder/Makefile b/contrib/randomDictBuilder/Makefile index a2aade23a..443f6f047 100644 --- a/contrib/randomDictBuilder/Makefile +++ b/contrib/randomDictBuilder/Makefile @@ -14,14 +14,14 @@ rand: ./main $(ARG) -main: random.o main.o libzstd.a - gcc random.o main.o libzstd.a -o main +main: main.o random.o libzstd.a + gcc main.o random.o libzstd.a -o main -main.o: main.c - gcc -c main.c -I ../../programs -I ../../lib/common -I ../../lib -I ../../lib/dictBuilder -I random.h +main.o: main.c $(PROGRAM_FILES) + gcc -c main.c $(PROGRAM_FILES) -I random.h -I ../../programs -I ../../lib/common -I ../../lib -I ../../lib/dictBuilder -random.o: $(PROGRAM_FILES) random.c - gcc -c $(PROGRAM_FILES) -I ../../programs -I ../../lib/common -I random.h random.c +random.o: random.c + gcc -c random.c -I random.h -I ../../lib/common -I ../../lib/dictBuilder libzstd.a: $(MAKE) -C ../../lib libzstd.a diff --git a/contrib/randomDictBuilder/main.c b/contrib/randomDictBuilder/main.c index cf0b94769..d9295aa9a 100644 --- a/contrib/randomDictBuilder/main.c +++ b/contrib/randomDictBuilder/main.c @@ -3,13 +3,45 @@ #include /* strcmp, strlen */ #include /* errno */ #include -#include "fileio.h" /* stdinmark, stdoutmark, ZSTD_EXTENSION */ #include "random.h" +#include "fileio.h" /* stdinmark, stdoutmark, ZSTD_EXTENSION */ +#include "platform.h" /* Large Files support */ #include "util.h" +#include "zdict.h" +/*-************************************* +* Console display +***************************************/ #define DISPLAY(...) fprintf(stderr, __VA_ARGS__) #define DISPLAYLEVEL(l, ...) if (displayLevel>=l) { DISPLAY(__VA_ARGS__); } +static const U64 g_refreshRate = SEC_TO_MICRO / 6; +static UTIL_time_t g_displayClock = UTIL_TIME_INITIALIZER; + +#define DISPLAYUPDATE(l, ...) { if (displayLevel>=l) { \ + if ((UTIL_clockSpanMicro(g_displayClock) > g_refreshRate) || (displayLevel>=4)) \ + { g_displayClock = UTIL_getTime(); DISPLAY(__VA_ARGS__); \ + if (displayLevel>=4) fflush(stderr); } } } + +/*-************************************* +* Exceptions +***************************************/ +#ifndef DEBUG +# define DEBUG 0 +#endif +#define DEBUGOUTPUT(...) if (DEBUG) DISPLAY(__VA_ARGS__); +#define EXM_THROW(error, ...) \ +{ \ + DEBUGOUTPUT("Error defined at %s, line %i : \n", __FILE__, __LINE__); \ + DISPLAY("Error %i : ", error); \ + DISPLAY(__VA_ARGS__); \ + DISPLAY("\n"); \ + exit(error); \ +} + +/*-************************************* +* Constants +***************************************/ static const unsigned g_defaultMaxDictSize = 110 KB; #define DEFAULT_CLEVEL 3 #define DEFAULT_INPUTFILE "" @@ -17,7 +49,33 @@ static const unsigned g_defaultMaxDictSize = 110 KB; #define DEFAULT_OUTPUTFILE "defaultDict" #define DEFAULT_DICTID 0 +#define SAMPLESIZE_MAX (128 KB) +#define RANDOM_MAX_SAMPLES_SIZE (sizeof(size_t) == 8 ? ((U32)-1) : ((U32)1 GB)) +#define RANDOM_MEMMULT 9 +static const size_t g_maxMemory = (sizeof(size_t) == 4) ? (2 GB - 64 MB) : ((size_t)(512 MB) << sizeof(size_t)); +#define NOISELENGTH 32 + + +/*-************************************* +* Structs +***************************************/ +typedef struct { + U64 totalSizeToLoad; + unsigned oneSampleTooLarge; + unsigned nbSamples; +} fileStats; + +typedef struct { + const void* srcBuffer; + const size_t *samplesSizes; + size_t nbSamples; +}sampleInfo; + + +/*-************************************* +* Commandline related functions +***************************************/ static unsigned readU32FromChar(const char** stringPtr) { const char errorMsg[] = "error: numeric value too large"; @@ -42,7 +100,6 @@ static unsigned readU32FromChar(const char** stringPtr) return result; } - /** longCommandWArg() : * check if *stringPtr is the same as longCommand. * If yes, @return 1 and advances *stringPtr to the position which immediately follows longCommand. @@ -56,6 +113,225 @@ static unsigned longCommandWArg(const char** stringPtr, const char* longCommand) return result; } +/* ******************************************************** +* File related operations +**********************************************************/ +/** loadFiles() : + * load samples from files listed in fileNamesTable into buffer. + * works even if buffer is too small to load all samples. + * Also provides the size of each sample into sampleSizes table + * which must be sized correctly, using DiB_fileStats(). + * @return : nb of samples effectively loaded into `buffer` + * *bufferSizePtr is modified, it provides the amount data loaded within buffer. + * sampleSizes is filled with the size of each sample. + */ +static unsigned loadFiles(void* buffer, size_t* bufferSizePtr, + size_t* sampleSizes, unsigned sstSize, + const char** fileNamesTable, unsigned nbFiles, size_t targetChunkSize, + unsigned displayLevel) +{ + char* const buff = (char*)buffer; + size_t pos = 0; + unsigned nbLoadedChunks = 0, fileIndex; + + for (fileIndex=0; fileIndex *bufferSizePtr-pos) break; + { size_t const readSize = fread(buff+pos, 1, toLoad, f); + if (readSize != toLoad) EXM_THROW(11, "Pb reading %s", fileName); + pos += readSize; + sampleSizes[nbLoadedChunks++] = toLoad; + remainingToLoad -= targetChunkSize; + if (nbLoadedChunks == sstSize) { /* no more space left in sampleSizes table */ + fileIndex = nbFiles; /* stop there */ + break; + } + if (toLoad < targetChunkSize) { + fseek(f, (long)(targetChunkSize - toLoad), SEEK_CUR); + } } } + fclose(f); + } + DISPLAYLEVEL(2, "\r%79s\r", ""); + *bufferSizePtr = pos; + DISPLAYLEVEL(4, "loaded : %u KB \n", (U32)(pos >> 10)) + return nbLoadedChunks; +} + +#define rotl32(x,r) ((x << r) | (x >> (32 - r))) +static U32 getRand(U32* src) +{ + static const U32 prime1 = 2654435761U; + static const U32 prime2 = 2246822519U; + U32 rand32 = *src; + rand32 *= prime1; + rand32 ^= prime2; + rand32 = rotl32(rand32, 13); + *src = rand32; + return rand32 >> 5; +} + +/* shuffle() : + * shuffle a table of file names in a semi-random way + * It improves dictionary quality by reducing "locality" impact, so if sample set is very large, + * it will load random elements from it, instead of just the first ones. */ +static void shuffle(const char** fileNamesTable, unsigned nbFiles) { + U32 seed = 0xFD2FB528; + unsigned i; + for (i = nbFiles - 1; i > 0; --i) { + unsigned const j = getRand(&seed) % (i + 1); + const char* const tmp = fileNamesTable[j]; + fileNamesTable[j] = fileNamesTable[i]; + fileNamesTable[i] = tmp; + } +} + + +/*-******************************************************** +* Dictionary training functions +**********************************************************/ +static size_t findMaxMem(unsigned long long requiredMem) +{ + size_t const step = 8 MB; + void* testmem = NULL; + + requiredMem = (((requiredMem >> 23) + 1) << 23); + requiredMem += step; + if (requiredMem > g_maxMemory) requiredMem = g_maxMemory; + + while (!testmem) { + testmem = malloc((size_t)requiredMem); + requiredMem -= step; + } + + free(testmem); + return (size_t)requiredMem; +} + +static void saveDict(const char* dictFileName, + const void* buff, size_t buffSize) +{ + FILE* const f = fopen(dictFileName, "wb"); + if (f==NULL) EXM_THROW(3, "cannot open %s ", dictFileName); + + { size_t const n = fwrite(buff, 1, buffSize, f); + if (n!=buffSize) EXM_THROW(4, "%s : write error", dictFileName) } + + { size_t const n = (size_t)fclose(f); + if (n!=0) EXM_THROW(5, "%s : flush error", dictFileName) } +} + +/*! getFileStats() : + * Given a list of files, and a chunkSize (0 == no chunk, whole files) + * provides the amount of data to be loaded and the resulting nb of samples. + * This is useful primarily for allocation purpose => sample buffer, and sample sizes table. + */ +static fileStats getFileStats(const char** fileNamesTable, unsigned nbFiles, size_t chunkSize, unsigned displayLevel) +{ + fileStats fs; + unsigned n; + memset(&fs, 0, sizeof(fs)); + for (n=0; n 2*SAMPLESIZE_MAX); + fs.nbSamples += nbSamples; + } + DISPLAYLEVEL(4, "Preparing to load : %u KB \n", (U32)(fs.totalSizeToLoad >> 10)); + return fs; +} + +int RANDOM_trainFromFiles(const char* dictFileName, sampleInfo *info, unsigned maxDictSize, + ZDICT_random_params_t *params){ + unsigned const displayLevel = params->zParams.notificationLevel; + void* const dictBuffer = malloc(maxDictSize); + + int result = 0; + + /* Checks */ + if (!dictBuffer) + EXM_THROW(12, "not enough memory for trainFromFiles"); /* should not happen */ + + { size_t dictSize; + dictSize = ZDICT_trainFromBuffer_random(dictBuffer, maxDictSize, info->srcBuffer, + info->samplesSizes, info->nbSamples, *params); + DISPLAYLEVEL(2, "k=%u\n", params->k); + if (ZDICT_isError(dictSize)) { + DISPLAYLEVEL(1, "dictionary training failed : %s \n", ZDICT_getErrorName(dictSize)); /* should not happen */ + result = 1; + free(dictBuffer); + } + /* save dict */ + DISPLAYLEVEL(2, "Save dictionary of size %u into file %s \n", (U32)dictSize, dictFileName); + saveDict(dictFileName, dictBuffer, dictSize); + } + + /* clean up */ + free(dictBuffer); + return result; +} + +sampleInfo* getSampleInfo(const char** fileNamesTable, + unsigned nbFiles, size_t chunkSize, unsigned maxDictSize, const unsigned displayLevel){ + fileStats const fs = getFileStats(fileNamesTable, nbFiles, chunkSize, displayLevel); + size_t* const sampleSizes = (size_t*)malloc(fs.nbSamples * sizeof(size_t)); + size_t const memMult = RANDOM_MEMMULT; + size_t const maxMem = findMaxMem(fs.totalSizeToLoad * memMult) / memMult; + size_t loadedSize = (size_t) MIN ((unsigned long long)maxMem, fs.totalSizeToLoad); + void* const srcBuffer = malloc(loadedSize+NOISELENGTH); + + /* Checks */ + if ((!sampleSizes) || (!srcBuffer)) + EXM_THROW(12, "not enough memory for trainFromFiles"); /* should not happen */ + if (fs.oneSampleTooLarge) { + DISPLAYLEVEL(2, "! Warning : some sample(s) are very large \n"); + DISPLAYLEVEL(2, "! Note that dictionary is only useful for small samples. \n"); + DISPLAYLEVEL(2, "! As a consequence, only the first %u bytes of each sample are loaded \n", SAMPLESIZE_MAX); + } + if (fs.nbSamples < 5) { + DISPLAYLEVEL(2, "! Warning : nb of samples too low for proper processing ! \n"); + DISPLAYLEVEL(2, "! Please provide _one file per sample_. \n"); + DISPLAYLEVEL(2, "! Alternatively, split files into fixed-size blocks representative of samples, with -B# \n"); + EXM_THROW(14, "nb of samples too low"); /* we now clearly forbid this case */ + } + if (fs.totalSizeToLoad < (unsigned long long)(8 * maxDictSize)) { + DISPLAYLEVEL(2, "! Warning : data size of samples too small for target dictionary size \n"); + DISPLAYLEVEL(2, "! Samples should be about 100x larger than target dictionary size \n"); + } + + /* init */ + if (loadedSize < fs.totalSizeToLoad) + DISPLAYLEVEL(1, "Not enough memory; training on %u MB only...\n", (unsigned)(loadedSize >> 20)); + + /* Load input buffer */ + DISPLAYLEVEL(3, "Shuffling input files\n"); + shuffle(fileNamesTable, nbFiles); + nbFiles = loadFiles(srcBuffer, &loadedSize, sampleSizes, fs.nbSamples, fileNamesTable, nbFiles, chunkSize, displayLevel); + + sampleInfo *info = (sampleInfo *)malloc(sizeof(sampleInfo)); + + info->nbSamples = fs.nbSamples; + info->samplesSizes = sampleSizes; + info->srcBuffer = srcBuffer; + + return info; +} + + int main(int argCount, const char* argv[]) { @@ -63,7 +339,7 @@ int main(int argCount, const char* argv[]) const char* programName = argv[0]; int operationResult = 0; - /* Initialize parameters with default value */ + unsigned cLevel = DEFAULT_CLEVEL; char* inputFile = DEFAULT_INPUTFILE; unsigned k = DEFAULT_k; char* outputFile = DEFAULT_OUTPUTFILE; @@ -76,10 +352,10 @@ int main(int argCount, const char* argv[]) for (int i = 1; i < argCount; i++) { const char* argument = argv[i]; if (longCommandWArg(&argument, "k=")) { k = readU32FromChar(&argument); continue; } + if (longCommandWArg(&argument, "c=")) { cLevel = readU32FromChar(&argument); continue; } if (longCommandWArg(&argument, "dictID=")) { dictID = readU32FromChar(&argument); continue; } if (longCommandWArg(&argument, "maxdict=")) { maxDictSize = readU32FromChar(&argument); continue; } if (longCommandWArg(&argument, "in=")) { - /* Allow multiple input files */ inputFile = malloc(strlen(argument) + 1); strcpy(inputFile, argument); filenameTable[filenameIdx] = inputFile; @@ -96,12 +372,6 @@ int main(int argCount, const char* argv[]) return operationResult; } - if (maxDictSize == 0) { - DISPLAYLEVEL(1, "maxDictSize should not be 0.\n"); - operationResult = 1; - return operationResult; - } - char* fileNamesBuf = NULL; unsigned fileNamesNb = filenameIdx; int followLinks = 0; @@ -119,12 +389,15 @@ int main(int argCount, const char* argv[]) ZDICT_random_params_t params; ZDICT_params_t zParams; - zParams.compressionLevel = DEFAULT_CLEVEL; + zParams.compressionLevel = cLevel; zParams.notificationLevel = displayLevel; zParams.dictID = dictID; params.zParams = zParams; params.k = k; - operationResult = RANDOM_trainFromFiles(outputFile, maxDictSize, filenameTable, filenameIdx, blockSize, ¶ms); + sampleInfo* info= getSampleInfo(filenameTable, + filenameIdx, blockSize, maxDictSize, zParams.notificationLevel); + operationResult = RANDOM_trainFromFiles(outputFile, info, maxDictSize, ¶ms); + return operationResult; } diff --git a/contrib/randomDictBuilder/random.c b/contrib/randomDictBuilder/random.c index a59427ba4..96c02389f 100644 --- a/contrib/randomDictBuilder/random.c +++ b/contrib/randomDictBuilder/random.c @@ -5,24 +5,12 @@ #include /* malloc, free, qsort */ #include /* memset */ #include /* clock */ -#include "zstd_internal.h" /* includes zstd.h */ +#include "random.h" +#include "util.h" /* UTIL_getFileSize, UTIL_getTotalFileSize */ #ifndef ZDICT_STATIC_LINKING_ONLY #define ZDICT_STATIC_LINKING_ONLY #endif -#include "random.h" -#include "platform.h" /* Large Files support */ -#include "util.h" /* UTIL_getFileSize, UTIL_getTotalFileSize */ - -/*-************************************* -* Constants -***************************************/ -#define SAMPLESIZE_MAX (128 KB) -#define RANDOM_MAX_SAMPLES_SIZE (sizeof(size_t) == 8 ? ((U32)-1) : ((U32)1 GB)) -#define RANDOM_MEMMULT 9 -static const size_t g_maxMemory = (sizeof(size_t) == 4) ? (2 GB - 64 MB) : ((size_t)(512 MB) << sizeof(size_t)); - -#define NOISELENGTH 32 -#define DEFAULT_K 200 +#include "zdict.h" /*-************************************* * Console display @@ -30,179 +18,16 @@ static const size_t g_maxMemory = (sizeof(size_t) == 4) ? (2 GB - 64 MB) : ((siz #define DISPLAY(...) fprintf(stderr, __VA_ARGS__) #define DISPLAYLEVEL(l, ...) if (displayLevel>=l) { DISPLAY(__VA_ARGS__); } -static const U64 g_refreshRate = SEC_TO_MICRO / 6; -static UTIL_time_t g_displayClock = UTIL_TIME_INITIALIZER; - -#define DISPLAYUPDATE(l, ...) { if (displayLevel>=l) { \ - if ((UTIL_clockSpanMicro(g_displayClock) > g_refreshRate) || (displayLevel>=4)) \ - { g_displayClock = UTIL_getTime(); DISPLAY(__VA_ARGS__); \ - if (displayLevel>=4) fflush(stderr); } } } - - -/*-************************************* -* Exceptions -***************************************/ -#ifndef DEBUG -# define DEBUG 0 -#endif -#define DEBUGOUTPUT(...) if (DEBUG) DISPLAY(__VA_ARGS__); -#define EXM_THROW(error, ...) \ -{ \ - DEBUGOUTPUT("Error defined at %s, line %i : \n", __FILE__, __LINE__); \ - DISPLAY("Error %i : ", error); \ - DISPLAY(__VA_ARGS__); \ - DISPLAY("\n"); \ - exit(error); \ -} - - -/* ******************************************************** -* File related operations -**********************************************************/ -/** loadFiles() : - * load samples from files listed in fileNamesTable into buffer. - * works even if buffer is too small to load all samples. - * Also provides the size of each sample into sampleSizes table - * which must be sized correctly, using DiB_fileStats(). - * @return : nb of samples effectively loaded into `buffer` - * *bufferSizePtr is modified, it provides the amount data loaded within buffer. - * sampleSizes is filled with the size of each sample. - */ -static unsigned loadFiles(void* buffer, size_t* bufferSizePtr, - size_t* sampleSizes, unsigned sstSize, - const char** fileNamesTable, unsigned nbFiles, size_t targetChunkSize, - unsigned displayLevel) -{ - char* const buff = (char*)buffer; - size_t pos = 0; - unsigned nbLoadedChunks = 0, fileIndex; - - for (fileIndex=0; fileIndex *bufferSizePtr-pos) break; - { size_t const readSize = fread(buff+pos, 1, toLoad, f); - if (readSize != toLoad) EXM_THROW(11, "Pb reading %s", fileName); - pos += readSize; - sampleSizes[nbLoadedChunks++] = toLoad; - remainingToLoad -= targetChunkSize; - if (nbLoadedChunks == sstSize) { /* no more space left in sampleSizes table */ - fileIndex = nbFiles; /* stop there */ - break; - } - if (toLoad < targetChunkSize) { - fseek(f, (long)(targetChunkSize - toLoad), SEEK_CUR); - } } } - fclose(f); - } - DISPLAYLEVEL(2, "\r%79s\r", ""); - *bufferSizePtr = pos; - DISPLAYLEVEL(4, "loaded : %u KB \n", (U32)(pos >> 10)) - return nbLoadedChunks; -} - - - -#define rotl32(x,r) ((x << r) | (x >> (32 - r))) -static U32 getRand(U32* src) -{ - static const U32 prime1 = 2654435761U; - static const U32 prime2 = 2246822519U; - U32 rand32 = *src; - rand32 *= prime1; - rand32 ^= prime2; - rand32 = rotl32(rand32, 13); - *src = rand32; - return rand32 >> 5; -} - - -/* shuffle() : - * shuffle a table of file names in a semi-random way - * It improves dictionary quality by reducing "locality" impact, so if sample set is very large, - * it will load random elements from it, instead of just the first ones. */ -static void shuffle(const char** fileNamesTable, unsigned nbFiles) { - U32 seed = 0xFD2FB528; - unsigned i; - for (i = nbFiles - 1; i > 0; --i) { - unsigned const j = getRand(&seed) % (i + 1); - const char* const tmp = fileNamesTable[j]; - fileNamesTable[j] = fileNamesTable[i]; - fileNamesTable[i] = tmp; - } -} - - - -/*-******************************************************** -* Dictionary training functions -**********************************************************/ -static size_t findMaxMem(unsigned long long requiredMem) -{ - size_t const step = 8 MB; - void* testmem = NULL; - - requiredMem = (((requiredMem >> 23) + 1) << 23); - requiredMem += step; - if (requiredMem > g_maxMemory) requiredMem = g_maxMemory; - - while (!testmem) { - testmem = malloc((size_t)requiredMem); - requiredMem -= step; - } - - free(testmem); - return (size_t)requiredMem; -} - -static void saveDict(const char* dictFileName, - const void* buff, size_t buffSize) -{ - FILE* const f = fopen(dictFileName, "wb"); - if (f==NULL) EXM_THROW(3, "cannot open %s ", dictFileName); - - { size_t const n = fwrite(buff, 1, buffSize, f); - if (n!=buffSize) EXM_THROW(4, "%s : write error", dictFileName) } - - { size_t const n = (size_t)fclose(f); - if (n!=0) EXM_THROW(5, "%s : flush error", dictFileName) } -} - -/*! getFileStats() : - * Given a list of files, and a chunkSize (0 == no chunk, whole files) - * provides the amount of data to be loaded and the resulting nb of samples. - * This is useful primarily for allocation purpose => sample buffer, and sample sizes table. - */ -static fileStats getFileStats(const char** fileNamesTable, unsigned nbFiles, size_t chunkSize, unsigned displayLevel) -{ - fileStats fs; - unsigned n; - memset(&fs, 0, sizeof(fs)); - for (n=0; n 2*SAMPLESIZE_MAX); - fs.nbSamples += nbSamples; - } - DISPLAYLEVEL(4, "Preparing to load : %u KB \n", (U32)(fs.totalSizeToLoad >> 10)); - return fs; -} - - +#define LOCALDISPLAYUPDATE(displayLevel, l, ...) \ + if (displayLevel >= l) { \ + if ((clock() - g_time > refreshRate) || (displayLevel >= 4)) { \ + g_time = clock(); \ + DISPLAY(__VA_ARGS__); \ + } \ + } +#define DISPLAYUPDATE(l, ...) LOCALDISPLAYUPDATE(displayLevel, l, __VA_ARGS__) +static const clock_t refreshRate = CLOCKS_PER_SEC * 15 / 100; +static clock_t g_time = 0; @@ -225,16 +50,14 @@ static size_t RANDOM_sum(const size_t *samplesSizes, unsigned nbSamples) { /** * Selects a random segment from totalSamplesSize - k + 1 possible segments */ -static RANDOM_segment_t RANDOM_selectSegment(const RANDOM_ctx_t *ctx, +static RANDOM_segment_t RANDOM_selectSegment(const size_t totalSamplesSize, ZDICT_random_params_t parameters) { const U32 k = parameters.k; RANDOM_segment_t segment; unsigned index; - /* Seed random number generator */ - srand((unsigned)time(NULL)); /* Randomly generate a number from 0 to sampleSizes - k */ - index = rand()%(ctx->totalSamplesSize - k + 1); + index = rand()%(totalSamplesSize - k + 1); /* inclusive */ segment.begin = index; @@ -261,65 +84,11 @@ static int RANDOM_checkParameters(ZDICT_random_params_t parameters, size_t maxDi } -/** - * Clean up a context initialized with `RANDOM_ctx_init()`. - */ -static void RANDOM_ctx_destroy(RANDOM_ctx_t *ctx) { - if (!ctx) { - return; - } - if (ctx->offsets) { - free(ctx->offsets); - ctx->offsets = NULL; - } -} - - -/** - * Prepare a context for dictionary building. - * Returns 1 on success or zero on error. - * The context must be destroyed with `RANDOM_ctx_destroy()`. - */ -static int RANDOM_ctx_init(RANDOM_ctx_t *ctx, const void *samplesBuffer, - const size_t *samplesSizes, unsigned nbSamples) { - const BYTE *const samples = (const BYTE *)samplesBuffer; - const size_t totalSamplesSize = RANDOM_sum(samplesSizes, nbSamples); - const int displayLevel = 2; - /* Checks */ - if (totalSamplesSize >= (size_t)RANDOM_MAX_SAMPLES_SIZE) { - DISPLAYLEVEL(1, "Total samples size is too large (%u MB), maximum size is %u MB\n", - (U32)(totalSamplesSize>>20), (RANDOM_MAX_SAMPLES_SIZE >> 20)); - return 0; - } - memset(ctx, 0, sizeof(*ctx)); - DISPLAYLEVEL(1, "Building dictionary from %u samples of total size %u\n", nbSamples, - (U32)totalSamplesSize); - ctx->samples = samples; - ctx->samplesSizes = samplesSizes; - ctx->nbSamples = nbSamples; - ctx->offsets = (size_t *)malloc((nbSamples + 1) * sizeof(size_t)); - ctx->totalSamplesSize = (U32)totalSamplesSize; - if (!ctx->offsets) { - DISPLAYLEVEL(1, "Failed to allocate buffer for offsets\n"); - RANDOM_ctx_destroy(ctx); - return 0; - } - { - U32 i; - ctx->offsets[0] = 0; - for (i = 1; i <= nbSamples; ++i) { - ctx->offsets[i] = ctx->offsets[i - 1] + samplesSizes[i - 1]; - } - } - return 1; -} - - /** * Given the prepared context build the dictionary. */ -static size_t RANDOM_buildDictionary(const RANDOM_ctx_t *ctx, void *dictBuffer, - size_t dictBufferCapacity, +static size_t RANDOM_buildDictionary(const size_t totalSamplesSize, const BYTE *samples, + void *dictBuffer, size_t dictBufferCapacity, ZDICT_random_params_t parameters) { BYTE *const dict = (BYTE *)dictBuffer; size_t tail = dictBufferCapacity; @@ -327,13 +96,13 @@ static size_t RANDOM_buildDictionary(const RANDOM_ctx_t *ctx, void *dictBuffer, while (tail > 0) { /* Select a segment */ - RANDOM_segment_t segment = RANDOM_selectSegment(ctx, parameters); + RANDOM_segment_t segment = RANDOM_selectSegment(totalSamplesSize, parameters); size_t segmentSize; segmentSize = MIN(segment.end - segment.begin + 1, tail); tail -= segmentSize; - memcpy(dict + tail, ctx->samples + segment.begin, segmentSize); + memcpy(dict + tail, samples + segment.begin, segmentSize); DISPLAYUPDATE( 2, "\r%u%% ", (U32)(((dictBufferCapacity - tail) * 100) / dictBufferCapacity)); @@ -342,6 +111,7 @@ static size_t RANDOM_buildDictionary(const RANDOM_ctx_t *ctx, void *dictBuffer, return tail; } + /*! ZDICT_trainFromBuffer_random(): * Train a dictionary from an array of samples using the RANDOM algorithm. * Samples must be stored concatenated in a single flat buffer `samplesBuffer`, @@ -356,7 +126,6 @@ ZDICTLIB_API size_t ZDICT_trainFromBuffer_random( ZDICT_random_params_t parameters) { const int displayLevel = parameters.zParams.notificationLevel; BYTE* const dict = (BYTE*)dictBuffer; - RANDOM_ctx_t ctx; /* Checks */ if (!RANDOM_checkParameters(parameters, dictBufferCapacity)) { DISPLAYLEVEL(1, "k is incorrect\n"); @@ -371,13 +140,12 @@ ZDICTLIB_API size_t ZDICT_trainFromBuffer_random( ZDICT_DICTSIZE_MIN); return ERROR(dstSize_tooSmall); } + const size_t totalSamplesSize = RANDOM_sum(samplesSizes, nbSamples); + const BYTE *const samples = (const BYTE *)samplesBuffer; - if (!RANDOM_ctx_init(&ctx, samplesBuffer, samplesSizes, nbSamples)) { - return ERROR(GENERIC); - } DISPLAYLEVEL(2, "Building dictionary\n"); { - const size_t tail = RANDOM_buildDictionary(&ctx, dictBuffer, dictBufferCapacity, parameters); + const size_t tail = RANDOM_buildDictionary(totalSamplesSize, samples, dictBuffer, dictBufferCapacity, parameters); const size_t dictSize = ZDICT_finalizeDictionary( dict, dictBufferCapacity, dict + tail, dictBufferCapacity - tail, samplesBuffer, samplesSizes, nbSamples, parameters.zParams); @@ -385,71 +153,6 @@ ZDICTLIB_API size_t ZDICT_trainFromBuffer_random( DISPLAYLEVEL(2, "Constructed dictionary of size %u\n", (U32)dictSize); } - RANDOM_ctx_destroy(&ctx); return dictSize; } } - - -int RANDOM_trainFromFiles(const char* dictFileName, unsigned maxDictSize, - const char** fileNamesTable, unsigned nbFiles, - size_t chunkSize, ZDICT_random_params_t *params){ - unsigned const displayLevel = params->zParams.notificationLevel; - void* const dictBuffer = malloc(maxDictSize); - fileStats const fs = getFileStats(fileNamesTable, nbFiles, chunkSize, displayLevel); - size_t* const sampleSizes = (size_t*)malloc(fs.nbSamples * sizeof(size_t)); - size_t const memMult = RANDOM_MEMMULT; - size_t const maxMem = findMaxMem(fs.totalSizeToLoad * memMult) / memMult; - size_t loadedSize = (size_t) MIN ((unsigned long long)maxMem, fs.totalSizeToLoad); - void* const srcBuffer = malloc(loadedSize+NOISELENGTH); - int result = 0; - - /* Checks */ - if ((!sampleSizes) || (!srcBuffer) || (!dictBuffer)) - EXM_THROW(12, "not enough memory for DiB_trainFiles"); /* should not happen */ - if (fs.oneSampleTooLarge) { - DISPLAYLEVEL(2, "! Warning : some sample(s) are very large \n"); - DISPLAYLEVEL(2, "! Note that dictionary is only useful for small samples. \n"); - DISPLAYLEVEL(2, "! As a consequence, only the first %u bytes of each sample are loaded \n", SAMPLESIZE_MAX); - } - if (fs.nbSamples < 5) { - DISPLAYLEVEL(2, "! Warning : nb of samples too low for proper processing ! \n"); - DISPLAYLEVEL(2, "! Please provide _one file per sample_. \n"); - DISPLAYLEVEL(2, "! Alternatively, split files into fixed-size blocks representative of samples, with -B# \n"); - EXM_THROW(14, "nb of samples too low"); /* we now clearly forbid this case */ - } - if (fs.totalSizeToLoad < (unsigned long long)(8 * maxDictSize)) { - DISPLAYLEVEL(2, "! Warning : data size of samples too small for target dictionary size \n"); - DISPLAYLEVEL(2, "! Samples should be about 100x larger than target dictionary size \n"); - } - - /* init */ - if (loadedSize < fs.totalSizeToLoad) - DISPLAYLEVEL(1, "Not enough memory; training on %u MB only...\n", (unsigned)(loadedSize >> 20)); - - /* Load input buffer */ - DISPLAYLEVEL(3, "Shuffling input files\n"); - shuffle(fileNamesTable, nbFiles); - nbFiles = loadFiles(srcBuffer, &loadedSize, sampleSizes, fs.nbSamples, fileNamesTable, nbFiles, chunkSize, displayLevel); - - { size_t dictSize; - dictSize = ZDICT_trainFromBuffer_random(dictBuffer, maxDictSize, srcBuffer, - sampleSizes, fs.nbSamples, *params); - DISPLAYLEVEL(2, "k=%u\n", params->k); - if (ZDICT_isError(dictSize)) { - DISPLAYLEVEL(1, "dictionary training failed : %s \n", ZDICT_getErrorName(dictSize)); /* should not happen */ - result = 1; - goto _cleanup; - } - /* save dict */ - DISPLAYLEVEL(2, "Save dictionary of size %u into file %s \n", (U32)dictSize, dictFileName); - saveDict(dictFileName, dictBuffer, dictSize); - } - - /* clean up */ -_cleanup: - free(srcBuffer); - free(sampleSizes); - free(dictBuffer); - return result; -} diff --git a/contrib/randomDictBuilder/random.h b/contrib/randomDictBuilder/random.h index 058796414..77529dafb 100644 --- a/contrib/randomDictBuilder/random.h +++ b/contrib/randomDictBuilder/random.h @@ -8,18 +8,6 @@ #endif #include "zdict.h" - -/************************************** -* Context -***************************************/ -typedef struct { - const BYTE *samples; - size_t *offsets; - const size_t *samplesSizes; - size_t nbSamples; - U32 totalSamplesSize; -} RANDOM_ctx_t; - /** * A segment is an inclusive range in the source. */ @@ -35,19 +23,8 @@ typedef struct { } ZDICT_random_params_t; -typedef struct { - U64 totalSizeToLoad; - unsigned oneSampleTooLarge; - unsigned nbSamples; -} fileStats; - ZDICTLIB_API size_t ZDICT_trainFromBuffer_random( void *dictBuffer, size_t dictBufferCapacity, const void *samplesBuffer, const size_t *samplesSizes, unsigned nbSamples, ZDICT_random_params_t parameters); - - -int RANDOM_trainFromFiles(const char* dictFileName, unsigned maxDictSize, - const char** fileNamesTable, unsigned nbFiles, - size_t chunkSize, ZDICT_random_params_t *params); From 1f7fa5cdd6555e22dfa8c2dc1f5c17293e703fe3 Mon Sep 17 00:00:00 2001 From: Jennifer Liu Date: Mon, 16 Jul 2018 16:31:59 -0700 Subject: [PATCH 053/372] Fix spacing and Edit Makefile (now run with make instead of make run) --- contrib/randomDictBuilder/Makefile | 13 +++++---- contrib/randomDictBuilder/README.md | 9 ++++--- contrib/randomDictBuilder/main.c | 42 ++++++++++++++--------------- contrib/randomDictBuilder/random.c | 9 ++++--- contrib/randomDictBuilder/random.h | 5 ++-- 5 files changed, 40 insertions(+), 38 deletions(-) diff --git a/contrib/randomDictBuilder/Makefile b/contrib/randomDictBuilder/Makefile index 443f6f047..77dd29337 100644 --- a/contrib/randomDictBuilder/Makefile +++ b/contrib/randomDictBuilder/Makefile @@ -4,16 +4,15 @@ TEST_INPUT := ../../lib TEST_OUTPUT := randomDict ARG := -all: main testrun test clean +all: main run clean -run: main rand clean +test: main testrun testshell clean -.PHONY: rand -rand: +.PHONY: run +run: echo "Building a random dictionary with given arguments" ./main $(ARG) - main: main.o random.o libzstd.a gcc main.o random.o libzstd.a -o main @@ -34,8 +33,8 @@ testrun: main zstd -be3 -D $(TEST_OUTPUT) -r $(TEST_INPUT) -q rm -f $(TEST_OUTPUT) -.PHONY: test -test: test.sh +.PHONY: testshell +testshell: test.sh sh test.sh echo "Finish running test.sh" diff --git a/contrib/randomDictBuilder/README.md b/contrib/randomDictBuilder/README.md index 09f1e8088..0e70d3dcc 100644 --- a/contrib/randomDictBuilder/README.md +++ b/contrib/randomDictBuilder/README.md @@ -7,11 +7,14 @@ Dictionary ID (dictID=#): nonnegative number; if not provided, default to 0 Maximum Dictionary Size (maxdict=#): positive number; in bytes, if not provided, default to 110KB Size of Randomly Selected Segment (k=#): positive number; in bytes; if not provided, default to 200 +###Running Test: +make test + ###Usage: -To build a random dictionary with the provided arguments: make run ARG= followed by arguments +To build a random dictionary with the provided arguments: make ARG= followed by arguments ### Examples: -make run ARG="in=../../lib/dictBuilder out=dict100 dictID=520" -make run ARG="in=../../lib/dictBuilder in=../../lib/compress" +make ARG="in=../../lib/dictBuilder out=dict100 dictID=520" +make ARG="in=../../lib/dictBuilder in=../../lib/compress" diff --git a/contrib/randomDictBuilder/main.c b/contrib/randomDictBuilder/main.c index d9295aa9a..e195188be 100644 --- a/contrib/randomDictBuilder/main.c +++ b/contrib/randomDictBuilder/main.c @@ -52,7 +52,8 @@ static const unsigned g_defaultMaxDictSize = 110 KB; #define SAMPLESIZE_MAX (128 KB) #define RANDOM_MAX_SAMPLES_SIZE (sizeof(size_t) == 8 ? ((U32)-1) : ((U32)1 GB)) #define RANDOM_MEMMULT 9 -static const size_t g_maxMemory = (sizeof(size_t) == 4) ? (2 GB - 64 MB) : ((size_t)(512 MB) << sizeof(size_t)); +static const size_t g_maxMemory = (sizeof(size_t) == 4) ? + (2 GB - 64 MB) : ((size_t)(512 MB) << sizeof(size_t)); #define NOISELENGTH 32 @@ -76,8 +77,7 @@ typedef struct { /*-************************************* * Commandline related functions ***************************************/ -static unsigned readU32FromChar(const char** stringPtr) -{ +static unsigned readU32FromChar(const char** stringPtr){ const char errorMsg[] = "error: numeric value too large"; unsigned result = 0; while ((**stringPtr >='0') && (**stringPtr <='9')) { @@ -105,8 +105,7 @@ static unsigned readU32FromChar(const char** stringPtr) * If yes, @return 1 and advances *stringPtr to the position which immediately follows longCommand. * @return 0 and doesn't modify *stringPtr otherwise. */ -static unsigned longCommandWArg(const char** stringPtr, const char* longCommand) -{ +static unsigned longCommandWArg(const char** stringPtr, const char* longCommand){ size_t const comSize = strlen(longCommand); int const result = !strncmp(*stringPtr, longCommand, comSize); if (result) *stringPtr += comSize; @@ -125,11 +124,9 @@ static unsigned longCommandWArg(const char** stringPtr, const char* longCommand) * *bufferSizePtr is modified, it provides the amount data loaded within buffer. * sampleSizes is filled with the size of each sample. */ -static unsigned loadFiles(void* buffer, size_t* bufferSizePtr, - size_t* sampleSizes, unsigned sstSize, - const char** fileNamesTable, unsigned nbFiles, size_t targetChunkSize, - unsigned displayLevel) -{ +static unsigned loadFiles(void* buffer, size_t* bufferSizePtr, size_t* sampleSizes, + unsigned sstSize, const char** fileNamesTable, unsigned nbFiles, + size_t targetChunkSize, unsigned displayLevel) { char* const buff = (char*)buffer; size_t pos = 0; unsigned nbLoadedChunks = 0, fileIndex; @@ -200,8 +197,7 @@ static void shuffle(const char** fileNamesTable, unsigned nbFiles) { /*-******************************************************** * Dictionary training functions **********************************************************/ -static size_t findMaxMem(unsigned long long requiredMem) -{ +static size_t findMaxMem(unsigned long long requiredMem) { size_t const step = 8 MB; void* testmem = NULL; @@ -219,8 +215,7 @@ static size_t findMaxMem(unsigned long long requiredMem) } static void saveDict(const char* dictFileName, - const void* buff, size_t buffSize) -{ + const void* buff, size_t buffSize) { FILE* const f = fopen(dictFileName, "wb"); if (f==NULL) EXM_THROW(3, "cannot open %s ", dictFileName); @@ -236,8 +231,8 @@ static void saveDict(const char* dictFileName, * provides the amount of data to be loaded and the resulting nb of samples. * This is useful primarily for allocation purpose => sample buffer, and sample sizes table. */ -static fileStats getFileStats(const char** fileNamesTable, unsigned nbFiles, size_t chunkSize, unsigned displayLevel) -{ +static fileStats getFileStats(const char** fileNamesTable, unsigned nbFiles, + size_t chunkSize, unsigned displayLevel) { fileStats fs; unsigned n; memset(&fs, 0, sizeof(fs)); @@ -255,8 +250,9 @@ static fileStats getFileStats(const char** fileNamesTable, unsigned nbFiles, siz return fs; } -int RANDOM_trainFromFiles(const char* dictFileName, sampleInfo *info, unsigned maxDictSize, - ZDICT_random_params_t *params){ +int RANDOM_trainFromFiles(const char* dictFileName, sampleInfo *info, + unsigned maxDictSize, + ZDICT_random_params_t *params) { unsigned const displayLevel = params->zParams.notificationLevel; void* const dictBuffer = malloc(maxDictSize); @@ -285,8 +281,8 @@ int RANDOM_trainFromFiles(const char* dictFileName, sampleInfo *info, unsigned m return result; } -sampleInfo* getSampleInfo(const char** fileNamesTable, - unsigned nbFiles, size_t chunkSize, unsigned maxDictSize, const unsigned displayLevel){ +sampleInfo* getSampleInfo(const char** fileNamesTable, unsigned nbFiles, size_t chunkSize, + unsigned maxDictSize, const unsigned displayLevel) { fileStats const fs = getFileStats(fileNamesTable, nbFiles, chunkSize, displayLevel); size_t* const sampleSizes = (size_t*)malloc(fs.nbSamples * sizeof(size_t)); size_t const memMult = RANDOM_MEMMULT; @@ -320,7 +316,8 @@ sampleInfo* getSampleInfo(const char** fileNamesTable, /* Load input buffer */ DISPLAYLEVEL(3, "Shuffling input files\n"); shuffle(fileNamesTable, nbFiles); - nbFiles = loadFiles(srcBuffer, &loadedSize, sampleSizes, fs.nbSamples, fileNamesTable, nbFiles, chunkSize, displayLevel); + nbFiles = loadFiles(srcBuffer, &loadedSize, sampleSizes, fs.nbSamples, + fileNamesTable, nbFiles, chunkSize, displayLevel); sampleInfo *info = (sampleInfo *)malloc(sizeof(sampleInfo)); @@ -376,7 +373,8 @@ int main(int argCount, const char* argv[]) unsigned fileNamesNb = filenameIdx; int followLinks = 0; const char** extendedFileList = NULL; - extendedFileList = UTIL_createFileList(filenameTable, filenameIdx, &fileNamesBuf, &fileNamesNb, followLinks); + extendedFileList = UTIL_createFileList(filenameTable, filenameIdx, &fileNamesBuf, + &fileNamesNb, followLinks); if (extendedFileList) { unsigned u; for (u=0; u Date: Mon, 16 Jul 2018 18:59:18 -0700 Subject: [PATCH 054/372] Remove CLevel cli option which was accidentally added back in the last commit --- contrib/randomDictBuilder/main.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/contrib/randomDictBuilder/main.c b/contrib/randomDictBuilder/main.c index e195188be..e66f28478 100644 --- a/contrib/randomDictBuilder/main.c +++ b/contrib/randomDictBuilder/main.c @@ -336,7 +336,6 @@ int main(int argCount, const char* argv[]) const char* programName = argv[0]; int operationResult = 0; - unsigned cLevel = DEFAULT_CLEVEL; char* inputFile = DEFAULT_INPUTFILE; unsigned k = DEFAULT_k; char* outputFile = DEFAULT_OUTPUTFILE; @@ -349,7 +348,6 @@ int main(int argCount, const char* argv[]) for (int i = 1; i < argCount; i++) { const char* argument = argv[i]; if (longCommandWArg(&argument, "k=")) { k = readU32FromChar(&argument); continue; } - if (longCommandWArg(&argument, "c=")) { cLevel = readU32FromChar(&argument); continue; } if (longCommandWArg(&argument, "dictID=")) { dictID = readU32FromChar(&argument); continue; } if (longCommandWArg(&argument, "maxdict=")) { maxDictSize = readU32FromChar(&argument); continue; } if (longCommandWArg(&argument, "in=")) { @@ -387,7 +385,7 @@ int main(int argCount, const char* argv[]) ZDICT_random_params_t params; ZDICT_params_t zParams; - zParams.compressionLevel = cLevel; + zParams.compressionLevel = DEFAULT_CLEVEL; zParams.notificationLevel = displayLevel; zParams.dictID = dictID; params.zParams = zParams; From 53e1f0504e077f90ecaea3a0bc18327177fd57ee Mon Sep 17 00:00:00 2001 From: cyan4973 Date: Tue, 17 Jul 2018 14:39:44 +0200 Subject: [PATCH 055/372] zstdmt debug traces compatibles with mingw since mingw does not have `sys/times.h`, remove this path when detecting mingw compilation. --- lib/compress/zstdmt_compress.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/compress/zstdmt_compress.c b/lib/compress/zstdmt_compress.c index 6daedca8b..d5193d52a 100644 --- a/lib/compress/zstdmt_compress.c +++ b/lib/compress/zstdmt_compress.c @@ -37,7 +37,9 @@ #define ZSTD_RESIZE_SEQPOOL 0 /* ====== Debug ====== */ -#if defined(DEBUGLEVEL) && (DEBUGLEVEL>=2) && !defined(_MSC_VER) +#if defined(DEBUGLEVEL) && (DEBUGLEVEL>=2) \ + && !defined(_MSC_VER) \ + && !defined(__MINGW32__) # include # include From 9597b438e931dcdd0cc8497bc330e9bcc495992a Mon Sep 17 00:00:00 2001 From: cyan4973 Date: Tue, 17 Jul 2018 18:52:57 +0200 Subject: [PATCH 056/372] fix #1241 Ensure that first input position is valid for a match even during first usage of context by starting reference at 1 (avoiding the problematic 0). --- lib/compress/zstd_compress.c | 3 +++ tests/fuzzer.c | 20 ++++++++++++++++++++ 2 files changed, 23 insertions(+) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index c66862524..8f63eb953 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -1032,6 +1032,9 @@ ZSTD_reset_matchState(ZSTD_matchState_t* ms, ms->hashLog3 = hashLog3; memset(&ms->window, 0, sizeof(ms->window)); + ms->window.dictLimit = 1; /* start from 1, so that 1st position is valid */ + ms->window.lowLimit = 1; /* it ensures first and later CCtx usages compress the same */ + ms->window.nextSrc = ms->window.base + 1; /* see issue #1241 */ ZSTD_invalidateMatchState(ms); /* opt parser space */ diff --git a/tests/fuzzer.c b/tests/fuzzer.c index f7bacd5ca..b9d319ea2 100644 --- a/tests/fuzzer.c +++ b/tests/fuzzer.c @@ -411,6 +411,26 @@ static int basicUnitTests(U32 seed, double compressibility) } DISPLAYLEVEL(3, "OK \n"); + DISPLAYLEVEL(3, "test%3d : re-using a CCtx should compress the same : ", testNb++); + { int i; + for (i=0; i<20; i++) + ((char*)CNBuffer)[i] = i; /* ensure no match during initial section */ + memcpy((char*)CNBuffer + 20, CNBuffer, 10); /* create one match, starting from beginning of sample, which is the difficult case (see #1241) */ + for (i=1; i<=19; i++) { + ZSTD_CCtx* const cctx = ZSTD_createCCtx(); + size_t size1, size2; + DISPLAYLEVEL(5, "l%i ", i); + size1 = ZSTD_compressCCtx(cctx, compressedBuffer, compressedBufferSize, CNBuffer, 30, i); + CHECK_Z(size1); + size2 = ZSTD_compressCCtx(cctx, compressedBuffer, compressedBufferSize, CNBuffer, 30, i); + CHECK_Z(size2); + CHECK_EQ(size1, size2); + + ZSTD_freeCCtx(cctx); + } + } + DISPLAYLEVEL(3, "OK \n"); + DISPLAYLEVEL(3, "test%3d : ZSTD_CCtx_getParameter() : ", testNb++); { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); ZSTD_outBuffer out = {NULL, 0, 0}; From b8a81a988c5d81f5efdc4af9f28417f3fe71cfe8 Mon Sep 17 00:00:00 2001 From: cyan4973 Date: Tue, 17 Jul 2018 19:02:17 +0200 Subject: [PATCH 057/372] added a test to be played on travis to check a make variable construction --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index 80406064d..1fd7e6aad 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,6 +10,7 @@ addons: matrix: include: # Ubuntu 14.04 + - env: Cmd='make test' - env: Cmd='make gcc6install && CC=gcc-6 make -j all && make clean && CC=gcc-6 make clean uasan-test-zstd Date: Tue, 17 Jul 2018 19:19:48 +0200 Subject: [PATCH 058/372] fix make test on Linux MOREFLAGS+= doesn't work on Linux --- Makefile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 0f2fcc958..89c5a418d 100644 --- a/Makefile +++ b/Makefile @@ -63,8 +63,9 @@ zlibwrapper: $(MAKE) -C $(ZWRAPDIR) test .PHONY: test +test: MOREFLAGS += -g -DDEBUGLEVEL=1 -Werror test: - MOREFLAGS+="-g -DDEBUGLEVEL=1 -Werror" $(MAKE) -C $(PRGDIR) allVariants + MOREFLAGS="$(MOREFLAGS)" $(MAKE) -C $(PRGDIR) allVariants $(MAKE) -C $(TESTDIR) $@ .PHONY: shortest From 49acfaeaec44a25c4628a2512965445152e8776a Mon Sep 17 00:00:00 2001 From: Jennifer Liu Date: Tue, 17 Jul 2018 12:35:09 -0700 Subject: [PATCH 059/372] Move file loading functions to new file for access by benchmarking tool --- contrib/randomDictBuilder/Makefile | 11 +- contrib/randomDictBuilder/io.c | 243 +++++++++++++++++++++++++++++ contrib/randomDictBuilder/io.h | 33 ++++ contrib/randomDictBuilder/main.c | 215 +------------------------ 4 files changed, 290 insertions(+), 212 deletions(-) create mode 100644 contrib/randomDictBuilder/io.c create mode 100644 contrib/randomDictBuilder/io.h diff --git a/contrib/randomDictBuilder/Makefile b/contrib/randomDictBuilder/Makefile index 77dd29337..8360a4091 100644 --- a/contrib/randomDictBuilder/Makefile +++ b/contrib/randomDictBuilder/Makefile @@ -13,15 +13,18 @@ run: echo "Building a random dictionary with given arguments" ./main $(ARG) -main: main.o random.o libzstd.a - gcc main.o random.o libzstd.a -o main +main: main.o io.o random.o libzstd.a + gcc main.o io.o random.o libzstd.a -o main -main.o: main.c $(PROGRAM_FILES) - gcc -c main.c $(PROGRAM_FILES) -I random.h -I ../../programs -I ../../lib/common -I ../../lib -I ../../lib/dictBuilder +main.o: main.c + gcc -c main.c -I io.h -I random.h -I ../../programs -I ../../lib/common -I ../../lib -I ../../lib/dictBuilder random.o: random.c gcc -c random.c -I random.h -I ../../lib/common -I ../../lib/dictBuilder +io.o: io.c $(PROGRAM_FILES) + gcc -c io.c $(PROGRAM_FILES) -I io.h -I ../../programs -I ../../lib/common -I ../../lib -I ../../lib/dictBuilder + libzstd.a: $(MAKE) -C ../../lib libzstd.a mv ../../lib/libzstd.a . diff --git a/contrib/randomDictBuilder/io.c b/contrib/randomDictBuilder/io.c new file mode 100644 index 000000000..a5f71498d --- /dev/null +++ b/contrib/randomDictBuilder/io.c @@ -0,0 +1,243 @@ +#include /* fprintf */ +#include /* malloc, free, qsort */ +#include /* strcmp, strlen */ +#include /* errno */ +#include +#include "io.h" +#include "fileio.h" /* stdinmark, stdoutmark, ZSTD_EXTENSION */ +#include "platform.h" /* Large Files support */ +#include "util.h" +#include "zdict.h" + +/*-************************************* +* Console display +***************************************/ +#define DISPLAY(...) fprintf(stderr, __VA_ARGS__) +#define DISPLAYLEVEL(l, ...) if (displayLevel>=l) { DISPLAY(__VA_ARGS__); } + +static const U64 g_refreshRate = SEC_TO_MICRO / 6; +static UTIL_time_t g_displayClock = UTIL_TIME_INITIALIZER; + +#define DISPLAYUPDATE(l, ...) { if (displayLevel>=l) { \ + if ((UTIL_clockSpanMicro(g_displayClock) > g_refreshRate) || (displayLevel>=4)) \ + { g_displayClock = UTIL_getTime(); DISPLAY(__VA_ARGS__); \ + if (displayLevel>=4) fflush(stderr); } } } + +/*-************************************* +* Exceptions +***************************************/ +#ifndef DEBUG +# define DEBUG 0 +#endif +#define DEBUGOUTPUT(...) if (DEBUG) DISPLAY(__VA_ARGS__); +#define EXM_THROW(error, ...) \ +{ \ + DEBUGOUTPUT("Error defined at %s, line %i : \n", __FILE__, __LINE__); \ + DISPLAY("Error %i : ", error); \ + DISPLAY(__VA_ARGS__); \ + DISPLAY("\n"); \ + exit(error); \ +} + + +/*-************************************* +* Constants +***************************************/ + +#define SAMPLESIZE_MAX (128 KB) +#define RANDOM_MAX_SAMPLES_SIZE (sizeof(size_t) == 8 ? ((U32)-1) : ((U32)1 GB)) +#define RANDOM_MEMMULT 9 +static const size_t g_maxMemory = (sizeof(size_t) == 4) ? + (2 GB - 64 MB) : ((size_t)(512 MB) << sizeof(size_t)); + +#define NOISELENGTH 32 + + + +/* ******************************************************** +* File related operations +**********************************************************/ +/** loadFiles() : + * load samples from files listed in fileNamesTable into buffer. + * works even if buffer is too small to load all samples. + * Also provides the size of each sample into sampleSizes table + * which must be sized correctly, using DiB_fileStats(). + * @return : nb of samples effectively loaded into `buffer` + * *bufferSizePtr is modified, it provides the amount data loaded within buffer. + * sampleSizes is filled with the size of each sample. + */ +static unsigned loadFiles(void* buffer, size_t* bufferSizePtr, size_t* sampleSizes, + unsigned sstSize, const char** fileNamesTable, unsigned nbFiles, + size_t targetChunkSize, unsigned displayLevel) { + char* const buff = (char*)buffer; + size_t pos = 0; + unsigned nbLoadedChunks = 0, fileIndex; + + for (fileIndex=0; fileIndex *bufferSizePtr-pos) break; + { size_t const readSize = fread(buff+pos, 1, toLoad, f); + if (readSize != toLoad) EXM_THROW(11, "Pb reading %s", fileName); + pos += readSize; + sampleSizes[nbLoadedChunks++] = toLoad; + remainingToLoad -= targetChunkSize; + if (nbLoadedChunks == sstSize) { /* no more space left in sampleSizes table */ + fileIndex = nbFiles; /* stop there */ + break; + } + if (toLoad < targetChunkSize) { + fseek(f, (long)(targetChunkSize - toLoad), SEEK_CUR); + } } } + fclose(f); + } + DISPLAYLEVEL(2, "\r%79s\r", ""); + *bufferSizePtr = pos; + DISPLAYLEVEL(4, "loaded : %u KB \n", (U32)(pos >> 10)) + return nbLoadedChunks; +} + +#define rotl32(x,r) ((x << r) | (x >> (32 - r))) +static U32 getRand(U32* src) +{ + static const U32 prime1 = 2654435761U; + static const U32 prime2 = 2246822519U; + U32 rand32 = *src; + rand32 *= prime1; + rand32 ^= prime2; + rand32 = rotl32(rand32, 13); + *src = rand32; + return rand32 >> 5; +} + +/* shuffle() : + * shuffle a table of file names in a semi-random way + * It improves dictionary quality by reducing "locality" impact, so if sample set is very large, + * it will load random elements from it, instead of just the first ones. */ +static void shuffle(const char** fileNamesTable, unsigned nbFiles) { + U32 seed = 0xFD2FB528; + unsigned i; + for (i = nbFiles - 1; i > 0; --i) { + unsigned const j = getRand(&seed) % (i + 1); + const char* const tmp = fileNamesTable[j]; + fileNamesTable[j] = fileNamesTable[i]; + fileNamesTable[i] = tmp; + } +} + + +/*-******************************************************** +* Dictionary training functions +**********************************************************/ +static size_t findMaxMem(unsigned long long requiredMem) { + size_t const step = 8 MB; + void* testmem = NULL; + + requiredMem = (((requiredMem >> 23) + 1) << 23); + requiredMem += step; + if (requiredMem > g_maxMemory) requiredMem = g_maxMemory; + + while (!testmem) { + testmem = malloc((size_t)requiredMem); + requiredMem -= step; + } + + free(testmem); + return (size_t)requiredMem; +} + +void saveDict(const char* dictFileName, + const void* buff, size_t buffSize) { + FILE* const f = fopen(dictFileName, "wb"); + if (f==NULL) EXM_THROW(3, "cannot open %s ", dictFileName); + + { size_t const n = fwrite(buff, 1, buffSize, f); + if (n!=buffSize) EXM_THROW(4, "%s : write error", dictFileName) } + + { size_t const n = (size_t)fclose(f); + if (n!=0) EXM_THROW(5, "%s : flush error", dictFileName) } +} + +/*! getFileStats() : + * Given a list of files, and a chunkSize (0 == no chunk, whole files) + * provides the amount of data to be loaded and the resulting nb of samples. + * This is useful primarily for allocation purpose => sample buffer, and sample sizes table. + */ +static fileStats getFileStats(const char** fileNamesTable, unsigned nbFiles, + size_t chunkSize, unsigned displayLevel) { + fileStats fs; + unsigned n; + memset(&fs, 0, sizeof(fs)); + for (n=0; n 2*SAMPLESIZE_MAX); + fs.nbSamples += nbSamples; + } + DISPLAYLEVEL(4, "Preparing to load : %u KB \n", (U32)(fs.totalSizeToLoad >> 10)); + return fs; +} + + + + +sampleInfo* getSampleInfo(const char** fileNamesTable, unsigned nbFiles, size_t chunkSize, + unsigned maxDictSize, const unsigned displayLevel) { + fileStats const fs = getFileStats(fileNamesTable, nbFiles, chunkSize, displayLevel); + size_t* const sampleSizes = (size_t*)malloc(fs.nbSamples * sizeof(size_t)); + size_t const memMult = RANDOM_MEMMULT; + size_t const maxMem = findMaxMem(fs.totalSizeToLoad * memMult) / memMult; + size_t loadedSize = (size_t) MIN ((unsigned long long)maxMem, fs.totalSizeToLoad); + void* const srcBuffer = malloc(loadedSize+NOISELENGTH); + + /* Checks */ + if ((!sampleSizes) || (!srcBuffer)) + EXM_THROW(12, "not enough memory for trainFromFiles"); /* should not happen */ + if (fs.oneSampleTooLarge) { + DISPLAYLEVEL(2, "! Warning : some sample(s) are very large \n"); + DISPLAYLEVEL(2, "! Note that dictionary is only useful for small samples. \n"); + DISPLAYLEVEL(2, "! As a consequence, only the first %u bytes of each sample are loaded \n", SAMPLESIZE_MAX); + } + if (fs.nbSamples < 5) { + DISPLAYLEVEL(2, "! Warning : nb of samples too low for proper processing ! \n"); + DISPLAYLEVEL(2, "! Please provide _one file per sample_. \n"); + DISPLAYLEVEL(2, "! Alternatively, split files into fixed-size blocks representative of samples, with -B# \n"); + EXM_THROW(14, "nb of samples too low"); /* we now clearly forbid this case */ + } + if (fs.totalSizeToLoad < (unsigned long long)(8 * maxDictSize)) { + DISPLAYLEVEL(2, "! Warning : data size of samples too small for target dictionary size \n"); + DISPLAYLEVEL(2, "! Samples should be about 100x larger than target dictionary size \n"); + } + + /* init */ + if (loadedSize < fs.totalSizeToLoad) + DISPLAYLEVEL(1, "Not enough memory; training on %u MB only...\n", (unsigned)(loadedSize >> 20)); + + /* Load input buffer */ + DISPLAYLEVEL(3, "Shuffling input files\n"); + shuffle(fileNamesTable, nbFiles); + nbFiles = loadFiles(srcBuffer, &loadedSize, sampleSizes, fs.nbSamples, + fileNamesTable, nbFiles, chunkSize, displayLevel); + + sampleInfo *info = (sampleInfo *)malloc(sizeof(sampleInfo)); + + info->nbSamples = fs.nbSamples; + info->samplesSizes = sampleSizes; + info->srcBuffer = srcBuffer; + + return info; +} diff --git a/contrib/randomDictBuilder/io.h b/contrib/randomDictBuilder/io.h new file mode 100644 index 000000000..4b5639fe1 --- /dev/null +++ b/contrib/randomDictBuilder/io.h @@ -0,0 +1,33 @@ +#include /* fprintf */ +#include /* malloc, free, qsort */ +#include /* strcmp, strlen */ +#include /* errno */ +#include +#include "zstd_internal.h" /* includes zstd.h */ +#include "fileio.h" /* stdinmark, stdoutmark, ZSTD_EXTENSION */ +#include "platform.h" /* Large Files support */ +#include "util.h" +#include "zdict.h" + + +/*-************************************* +* Structs +***************************************/ +typedef struct { + U64 totalSizeToLoad; + unsigned oneSampleTooLarge; + unsigned nbSamples; +} fileStats; + +typedef struct { + const void* srcBuffer; + const size_t *samplesSizes; + size_t nbSamples; +}sampleInfo; + + +sampleInfo* getSampleInfo(const char** fileNamesTable, unsigned nbFiles, size_t chunkSize, + unsigned maxDictSize, const unsigned displayLevel); + + +void saveDict(const char* dictFileName, const void* buff, size_t buffSize); diff --git a/contrib/randomDictBuilder/main.c b/contrib/randomDictBuilder/main.c index e66f28478..34a9d99e2 100644 --- a/contrib/randomDictBuilder/main.c +++ b/contrib/randomDictBuilder/main.c @@ -4,11 +4,11 @@ #include /* errno */ #include #include "random.h" -#include "fileio.h" /* stdinmark, stdoutmark, ZSTD_EXTENSION */ -#include "platform.h" /* Large Files support */ +#include "io.h" #include "util.h" #include "zdict.h" + /*-************************************* * Console display ***************************************/ @@ -23,6 +23,7 @@ static UTIL_time_t g_displayClock = UTIL_TIME_INITIALIZER; { g_displayClock = UTIL_getTime(); DISPLAY(__VA_ARGS__); \ if (displayLevel>=4) fflush(stderr); } } } + /*-************************************* * Exceptions ***************************************/ @@ -39,6 +40,7 @@ static UTIL_time_t g_displayClock = UTIL_TIME_INITIALIZER; exit(error); \ } + /*-************************************* * Constants ***************************************/ @@ -49,29 +51,6 @@ static const unsigned g_defaultMaxDictSize = 110 KB; #define DEFAULT_OUTPUTFILE "defaultDict" #define DEFAULT_DICTID 0 -#define SAMPLESIZE_MAX (128 KB) -#define RANDOM_MAX_SAMPLES_SIZE (sizeof(size_t) == 8 ? ((U32)-1) : ((U32)1 GB)) -#define RANDOM_MEMMULT 9 -static const size_t g_maxMemory = (sizeof(size_t) == 4) ? - (2 GB - 64 MB) : ((size_t)(512 MB) << sizeof(size_t)); - -#define NOISELENGTH 32 - - -/*-************************************* -* Structs -***************************************/ -typedef struct { - U64 totalSizeToLoad; - unsigned oneSampleTooLarge; - unsigned nbSamples; -} fileStats; - -typedef struct { - const void* srcBuffer; - const size_t *samplesSizes; - size_t nbSamples; -}sampleInfo; /*-************************************* @@ -112,144 +91,11 @@ static unsigned longCommandWArg(const char** stringPtr, const char* longCommand) return result; } -/* ******************************************************** -* File related operations -**********************************************************/ -/** loadFiles() : - * load samples from files listed in fileNamesTable into buffer. - * works even if buffer is too small to load all samples. - * Also provides the size of each sample into sampleSizes table - * which must be sized correctly, using DiB_fileStats(). - * @return : nb of samples effectively loaded into `buffer` - * *bufferSizePtr is modified, it provides the amount data loaded within buffer. - * sampleSizes is filled with the size of each sample. - */ -static unsigned loadFiles(void* buffer, size_t* bufferSizePtr, size_t* sampleSizes, - unsigned sstSize, const char** fileNamesTable, unsigned nbFiles, - size_t targetChunkSize, unsigned displayLevel) { - char* const buff = (char*)buffer; - size_t pos = 0; - unsigned nbLoadedChunks = 0, fileIndex; - - for (fileIndex=0; fileIndex *bufferSizePtr-pos) break; - { size_t const readSize = fread(buff+pos, 1, toLoad, f); - if (readSize != toLoad) EXM_THROW(11, "Pb reading %s", fileName); - pos += readSize; - sampleSizes[nbLoadedChunks++] = toLoad; - remainingToLoad -= targetChunkSize; - if (nbLoadedChunks == sstSize) { /* no more space left in sampleSizes table */ - fileIndex = nbFiles; /* stop there */ - break; - } - if (toLoad < targetChunkSize) { - fseek(f, (long)(targetChunkSize - toLoad), SEEK_CUR); - } } } - fclose(f); - } - DISPLAYLEVEL(2, "\r%79s\r", ""); - *bufferSizePtr = pos; - DISPLAYLEVEL(4, "loaded : %u KB \n", (U32)(pos >> 10)) - return nbLoadedChunks; -} - -#define rotl32(x,r) ((x << r) | (x >> (32 - r))) -static U32 getRand(U32* src) -{ - static const U32 prime1 = 2654435761U; - static const U32 prime2 = 2246822519U; - U32 rand32 = *src; - rand32 *= prime1; - rand32 ^= prime2; - rand32 = rotl32(rand32, 13); - *src = rand32; - return rand32 >> 5; -} - -/* shuffle() : - * shuffle a table of file names in a semi-random way - * It improves dictionary quality by reducing "locality" impact, so if sample set is very large, - * it will load random elements from it, instead of just the first ones. */ -static void shuffle(const char** fileNamesTable, unsigned nbFiles) { - U32 seed = 0xFD2FB528; - unsigned i; - for (i = nbFiles - 1; i > 0; --i) { - unsigned const j = getRand(&seed) % (i + 1); - const char* const tmp = fileNamesTable[j]; - fileNamesTable[j] = fileNamesTable[i]; - fileNamesTable[i] = tmp; - } -} -/*-******************************************************** -* Dictionary training functions -**********************************************************/ -static size_t findMaxMem(unsigned long long requiredMem) { - size_t const step = 8 MB; - void* testmem = NULL; - - requiredMem = (((requiredMem >> 23) + 1) << 23); - requiredMem += step; - if (requiredMem > g_maxMemory) requiredMem = g_maxMemory; - - while (!testmem) { - testmem = malloc((size_t)requiredMem); - requiredMem -= step; - } - - free(testmem); - return (size_t)requiredMem; -} - -static void saveDict(const char* dictFileName, - const void* buff, size_t buffSize) { - FILE* const f = fopen(dictFileName, "wb"); - if (f==NULL) EXM_THROW(3, "cannot open %s ", dictFileName); - - { size_t const n = fwrite(buff, 1, buffSize, f); - if (n!=buffSize) EXM_THROW(4, "%s : write error", dictFileName) } - - { size_t const n = (size_t)fclose(f); - if (n!=0) EXM_THROW(5, "%s : flush error", dictFileName) } -} - -/*! getFileStats() : - * Given a list of files, and a chunkSize (0 == no chunk, whole files) - * provides the amount of data to be loaded and the resulting nb of samples. - * This is useful primarily for allocation purpose => sample buffer, and sample sizes table. - */ -static fileStats getFileStats(const char** fileNamesTable, unsigned nbFiles, - size_t chunkSize, unsigned displayLevel) { - fileStats fs; - unsigned n; - memset(&fs, 0, sizeof(fs)); - for (n=0; n 2*SAMPLESIZE_MAX); - fs.nbSamples += nbSamples; - } - DISPLAYLEVEL(4, "Preparing to load : %u KB \n", (U32)(fs.totalSizeToLoad >> 10)); - return fs; -} - +/*-************************************* +* RANDOM +***************************************/ int RANDOM_trainFromFiles(const char* dictFileName, sampleInfo *info, unsigned maxDictSize, ZDICT_random_params_t *params) { @@ -281,53 +127,6 @@ int RANDOM_trainFromFiles(const char* dictFileName, sampleInfo *info, return result; } -sampleInfo* getSampleInfo(const char** fileNamesTable, unsigned nbFiles, size_t chunkSize, - unsigned maxDictSize, const unsigned displayLevel) { - fileStats const fs = getFileStats(fileNamesTable, nbFiles, chunkSize, displayLevel); - size_t* const sampleSizes = (size_t*)malloc(fs.nbSamples * sizeof(size_t)); - size_t const memMult = RANDOM_MEMMULT; - size_t const maxMem = findMaxMem(fs.totalSizeToLoad * memMult) / memMult; - size_t loadedSize = (size_t) MIN ((unsigned long long)maxMem, fs.totalSizeToLoad); - void* const srcBuffer = malloc(loadedSize+NOISELENGTH); - - /* Checks */ - if ((!sampleSizes) || (!srcBuffer)) - EXM_THROW(12, "not enough memory for trainFromFiles"); /* should not happen */ - if (fs.oneSampleTooLarge) { - DISPLAYLEVEL(2, "! Warning : some sample(s) are very large \n"); - DISPLAYLEVEL(2, "! Note that dictionary is only useful for small samples. \n"); - DISPLAYLEVEL(2, "! As a consequence, only the first %u bytes of each sample are loaded \n", SAMPLESIZE_MAX); - } - if (fs.nbSamples < 5) { - DISPLAYLEVEL(2, "! Warning : nb of samples too low for proper processing ! \n"); - DISPLAYLEVEL(2, "! Please provide _one file per sample_. \n"); - DISPLAYLEVEL(2, "! Alternatively, split files into fixed-size blocks representative of samples, with -B# \n"); - EXM_THROW(14, "nb of samples too low"); /* we now clearly forbid this case */ - } - if (fs.totalSizeToLoad < (unsigned long long)(8 * maxDictSize)) { - DISPLAYLEVEL(2, "! Warning : data size of samples too small for target dictionary size \n"); - DISPLAYLEVEL(2, "! Samples should be about 100x larger than target dictionary size \n"); - } - - /* init */ - if (loadedSize < fs.totalSizeToLoad) - DISPLAYLEVEL(1, "Not enough memory; training on %u MB only...\n", (unsigned)(loadedSize >> 20)); - - /* Load input buffer */ - DISPLAYLEVEL(3, "Shuffling input files\n"); - shuffle(fileNamesTable, nbFiles); - nbFiles = loadFiles(srcBuffer, &loadedSize, sampleSizes, fs.nbSamples, - fileNamesTable, nbFiles, chunkSize, displayLevel); - - sampleInfo *info = (sampleInfo *)malloc(sizeof(sampleInfo)); - - info->nbSamples = fs.nbSamples; - info->samplesSizes = sampleSizes; - info->srcBuffer = srcBuffer; - - return info; -} - int main(int argCount, const char* argv[]) From e6fe4058388c820444a80d9d10aa5d840fab3c0c Mon Sep 17 00:00:00 2001 From: Jennifer Liu Date: Tue, 17 Jul 2018 12:42:53 -0700 Subject: [PATCH 060/372] Make test PHONY target --- contrib/randomDictBuilder/Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/contrib/randomDictBuilder/Makefile b/contrib/randomDictBuilder/Makefile index 8360a4091..678ff28a8 100644 --- a/contrib/randomDictBuilder/Makefile +++ b/contrib/randomDictBuilder/Makefile @@ -6,6 +6,7 @@ ARG := all: main run clean +.PHONY: test test: main testrun testshell clean .PHONY: run From 4e706d7f2cb79df257809b45c033b3bcf5822edf Mon Sep 17 00:00:00 2001 From: Nick Terrell Date: Tue, 17 Jul 2018 14:57:27 -0700 Subject: [PATCH 061/372] fileio: Error in compression on read errors We can write a corrupted file if the input file errors during a read. We should return a non-zero error code in this case. --- programs/fileio.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/programs/fileio.c b/programs/fileio.c index b4eed28d1..85367fdfc 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -797,6 +797,14 @@ FIO_compressZstdFrame(const cRess_t* ressPtr, } } while (directive != ZSTD_e_end); + if (ferror(srcFile)) { + EXM_THROW(26, "Read error : I/O error"); + } + if (fileSize != UTIL_FILESIZE_UNKNOWN && *readsize != fileSize) { + EXM_THROW(27, "Read error : Incomplete read : %llu / %llu B", + (unsigned long long)*readsize, (unsigned long long)fileSize); + } + return compressedfilesize; } From 896ff0644a2531a22edf78ea9cb6b58a4de9c77f Mon Sep 17 00:00:00 2001 From: Jennifer Liu Date: Tue, 17 Jul 2018 16:01:44 -0700 Subject: [PATCH 062/372] Fix deallocation problem and add documentation --- contrib/randomDictBuilder/io.c | 7 +++++++ contrib/randomDictBuilder/io.h | 17 +++++++++++++++++ contrib/randomDictBuilder/main.c | 20 +++++++++++--------- contrib/randomDictBuilder/random.c | 11 ++--------- contrib/randomDictBuilder/random.h | 9 ++++++++- 5 files changed, 45 insertions(+), 19 deletions(-) diff --git a/contrib/randomDictBuilder/io.c b/contrib/randomDictBuilder/io.c index a5f71498d..1c3eda589 100644 --- a/contrib/randomDictBuilder/io.c +++ b/contrib/randomDictBuilder/io.c @@ -241,3 +241,10 @@ sampleInfo* getSampleInfo(const char** fileNamesTable, unsigned nbFiles, size_t return info; } + + +void freeSampleInfo(sampleInfo *info) { + if (info->samplesSizes) free((void*)(info->samplesSizes)); + if (info->srcBuffer) free((void*)(info->srcBuffer)); + free(info); +} diff --git a/contrib/randomDictBuilder/io.h b/contrib/randomDictBuilder/io.h index 4b5639fe1..55967f76e 100644 --- a/contrib/randomDictBuilder/io.h +++ b/contrib/randomDictBuilder/io.h @@ -26,8 +26,25 @@ typedef struct { }sampleInfo; + +/*! getSampleInfo(): + * Load from input files and add samples to buffer + * @return: a sampleInfo struct containing infomation about buffer where samples are stored, + * size of each sample, and total number of samples + */ sampleInfo* getSampleInfo(const char** fileNamesTable, unsigned nbFiles, size_t chunkSize, unsigned maxDictSize, const unsigned displayLevel); + +/*! freeSampleInfo(): + * Free memory allocated for info + */ +void freeSampleInfo(sampleInfo *info); + + + +/*! saveDict(): + * Save data stored on buff to dictFileName + */ void saveDict(const char* dictFileName, const void* buff, size_t buffSize); diff --git a/contrib/randomDictBuilder/main.c b/contrib/randomDictBuilder/main.c index 34a9d99e2..1f12c7a4b 100644 --- a/contrib/randomDictBuilder/main.c +++ b/contrib/randomDictBuilder/main.c @@ -46,7 +46,6 @@ static UTIL_time_t g_displayClock = UTIL_TIME_INITIALIZER; ***************************************/ static const unsigned g_defaultMaxDictSize = 110 KB; #define DEFAULT_CLEVEL 3 -#define DEFAULT_INPUTFILE "" #define DEFAULT_k 200 #define DEFAULT_OUTPUTFILE "defaultDict" #define DEFAULT_DICTID 0 @@ -135,30 +134,29 @@ int main(int argCount, const char* argv[]) const char* programName = argv[0]; int operationResult = 0; - char* inputFile = DEFAULT_INPUTFILE; + /* Initialize arguments to default values */ unsigned k = DEFAULT_k; - char* outputFile = DEFAULT_OUTPUTFILE; + const char* outputFile = DEFAULT_OUTPUTFILE; unsigned dictID = DEFAULT_DICTID; unsigned maxDictSize = g_defaultMaxDictSize; + /* Initialize table to store input files */ const char** filenameTable = (const char**)malloc(argCount * sizeof(const char*)); unsigned filenameIdx = 0; + /* Parse arguments */ for (int i = 1; i < argCount; i++) { const char* argument = argv[i]; if (longCommandWArg(&argument, "k=")) { k = readU32FromChar(&argument); continue; } if (longCommandWArg(&argument, "dictID=")) { dictID = readU32FromChar(&argument); continue; } if (longCommandWArg(&argument, "maxdict=")) { maxDictSize = readU32FromChar(&argument); continue; } if (longCommandWArg(&argument, "in=")) { - inputFile = malloc(strlen(argument) + 1); - strcpy(inputFile, argument); - filenameTable[filenameIdx] = inputFile; + filenameTable[filenameIdx] = argument; filenameIdx++; continue; } if (longCommandWArg(&argument, "out=")) { - outputFile = malloc(strlen(argument) + 1); - strcpy(outputFile, argument); + outputFile = argument; continue; } DISPLAYLEVEL(1, "Incorrect parameters\n"); @@ -168,7 +166,7 @@ int main(int argCount, const char* argv[]) char* fileNamesBuf = NULL; unsigned fileNamesNb = filenameIdx; - int followLinks = 0; + int followLinks = 0; /* follow directory recursively */ const char** extendedFileList = NULL; extendedFileList = UTIL_createFileList(filenameTable, filenameIdx, &fileNamesBuf, &fileNamesNb, followLinks); @@ -194,5 +192,9 @@ int main(int argCount, const char* argv[]) filenameIdx, blockSize, maxDictSize, zParams.notificationLevel); operationResult = RANDOM_trainFromFiles(outputFile, info, maxDictSize, ¶ms); + /* Free allocated memory */ + UTIL_freeFileList(extendedFileList, fileNamesBuf); + freeSampleInfo(info); + return operationResult; } diff --git a/contrib/randomDictBuilder/random.c b/contrib/randomDictBuilder/random.c index cfed14a4a..34aec39ea 100644 --- a/contrib/randomDictBuilder/random.c +++ b/contrib/randomDictBuilder/random.c @@ -113,15 +113,8 @@ static size_t RANDOM_buildDictionary(const size_t totalSamplesSize, const BYTE * } -/*! ZDICT_trainFromBuffer_random(): - * Train a dictionary from an array of samples using the RANDOM algorithm. - * Samples must be stored concatenated in a single flat buffer `samplesBuffer`, - * supplied with an array of sizes `samplesSizes`, providing the size of each - * sample, in order. - * The resulting dictionary will be saved into `dictBuffer`. - * @return: size of dictionary stored into `dictBuffer` (<= `dictBufferCapacity`) - * or an error code, which can be tested with ZDICT_isError(). - */ + + ZDICTLIB_API size_t ZDICT_trainFromBuffer_random( void *dictBuffer, size_t dictBufferCapacity, const void *samplesBuffer, const size_t *samplesSizes, unsigned nbSamples, diff --git a/contrib/randomDictBuilder/random.h b/contrib/randomDictBuilder/random.h index b6696323a..c3146f861 100644 --- a/contrib/randomDictBuilder/random.h +++ b/contrib/randomDictBuilder/random.h @@ -23,7 +23,14 @@ typedef struct { } ZDICT_random_params_t; - +/*! ZDICT_trainFromBuffer_random(): + * Train a dictionary from an array of samples. + * Samples must be stored concatenated in a single flat buffer `samplesBuffer`, + * supplied with an array of sizes `samplesSizes`, providing the size of each sample, in order. + * The resulting dictionary will be saved into `dictBuffer`. + * @return: size of dictionary stored into `dictBuffer` (<= `dictBufferCapacity`) + * or an error code, which can be tested with ZDICT_isError(). + */ ZDICTLIB_API size_t ZDICT_trainFromBuffer_random( void *dictBuffer, size_t dictBufferCapacity, const void *samplesBuffer, const size_t *samplesSizes, unsigned nbSamples, ZDICT_random_params_t parameters); From ce09fb723d1311e62c920430fb14634e9b67dd70 Mon Sep 17 00:00:00 2001 From: Jennifer Liu Date: Tue, 17 Jul 2018 16:13:40 -0700 Subject: [PATCH 063/372] Update freeSampleInfo --- contrib/randomDictBuilder/io.c | 1 + 1 file changed, 1 insertion(+) diff --git a/contrib/randomDictBuilder/io.c b/contrib/randomDictBuilder/io.c index 1c3eda589..67c40858c 100644 --- a/contrib/randomDictBuilder/io.c +++ b/contrib/randomDictBuilder/io.c @@ -244,6 +244,7 @@ sampleInfo* getSampleInfo(const char** fileNamesTable, unsigned nbFiles, size_t void freeSampleInfo(sampleInfo *info) { + if (!info) return; if (info->samplesSizes) free((void*)(info->samplesSizes)); if (info->srcBuffer) free((void*)(info->srcBuffer)); free(info); From 7d1bc9cc8c79f7412c44835b910945792b2238c7 Mon Sep 17 00:00:00 2001 From: cyan4973 Date: Wed, 18 Jul 2018 16:10:23 +0200 Subject: [PATCH 064/372] fix minor conversion warning --- tests/fuzzer.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/fuzzer.c b/tests/fuzzer.c index b9d319ea2..c411d4177 100644 --- a/tests/fuzzer.c +++ b/tests/fuzzer.c @@ -414,7 +414,7 @@ static int basicUnitTests(U32 seed, double compressibility) DISPLAYLEVEL(3, "test%3d : re-using a CCtx should compress the same : ", testNb++); { int i; for (i=0; i<20; i++) - ((char*)CNBuffer)[i] = i; /* ensure no match during initial section */ + ((char*)CNBuffer)[i] = (char)i; /* ensure no match during initial section */ memcpy((char*)CNBuffer + 20, CNBuffer, 10); /* create one match, starting from beginning of sample, which is the difficult case (see #1241) */ for (i=1; i<=19; i++) { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); From 52e7cf0e405ac6eb827322b607d094125646bbfb Mon Sep 17 00:00:00 2001 From: Jennifer Liu Date: Wed, 18 Jul 2018 10:40:13 -0700 Subject: [PATCH 065/372] Add cleanup to trainfromFiles and move RANDOM_segment_t declaration --- contrib/randomDictBuilder/main.c | 3 ++- contrib/randomDictBuilder/random.c | 9 +++++++++ contrib/randomDictBuilder/random.h | 7 ------- 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/contrib/randomDictBuilder/main.c b/contrib/randomDictBuilder/main.c index 1f12c7a4b..36c4326b6 100644 --- a/contrib/randomDictBuilder/main.c +++ b/contrib/randomDictBuilder/main.c @@ -114,7 +114,7 @@ int RANDOM_trainFromFiles(const char* dictFileName, sampleInfo *info, if (ZDICT_isError(dictSize)) { DISPLAYLEVEL(1, "dictionary training failed : %s \n", ZDICT_getErrorName(dictSize)); /* should not happen */ result = 1; - free(dictBuffer); + goto _cleanup; } /* save dict */ DISPLAYLEVEL(2, "Save dictionary of size %u into file %s \n", (U32)dictSize, dictFileName); @@ -122,6 +122,7 @@ int RANDOM_trainFromFiles(const char* dictFileName, sampleInfo *info, } /* clean up */ +_cleanup: free(dictBuffer); return result; } diff --git a/contrib/randomDictBuilder/random.c b/contrib/randomDictBuilder/random.c index 34aec39ea..5276bea96 100644 --- a/contrib/randomDictBuilder/random.c +++ b/contrib/randomDictBuilder/random.c @@ -47,6 +47,15 @@ static size_t RANDOM_sum(const size_t *samplesSizes, unsigned nbSamples) { } +/** + * A segment is an inclusive range in the source. + */ +typedef struct { + U32 begin; + U32 end; +} RANDOM_segment_t; + + /** * Selects a random segment from totalSamplesSize - k + 1 possible segments */ diff --git a/contrib/randomDictBuilder/random.h b/contrib/randomDictBuilder/random.h index c3146f861..352775f95 100644 --- a/contrib/randomDictBuilder/random.h +++ b/contrib/randomDictBuilder/random.h @@ -8,13 +8,6 @@ #endif #include "zdict.h" -/** - * A segment is an inclusive range in the source. - */ -typedef struct { - U32 begin; - U32 end; -} RANDOM_segment_t; typedef struct { From 5bb46a898e6565e5bc1ee861999384f806f83831 Mon Sep 17 00:00:00 2001 From: Jennifer Liu Date: Wed, 18 Jul 2018 12:15:49 -0700 Subject: [PATCH 066/372] Rename cleanup --- contrib/randomDictBuilder/main.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/contrib/randomDictBuilder/main.c b/contrib/randomDictBuilder/main.c index 36c4326b6..4751a9e1c 100644 --- a/contrib/randomDictBuilder/main.c +++ b/contrib/randomDictBuilder/main.c @@ -114,7 +114,7 @@ int RANDOM_trainFromFiles(const char* dictFileName, sampleInfo *info, if (ZDICT_isError(dictSize)) { DISPLAYLEVEL(1, "dictionary training failed : %s \n", ZDICT_getErrorName(dictSize)); /* should not happen */ result = 1; - goto _cleanup; + goto _done; } /* save dict */ DISPLAYLEVEL(2, "Save dictionary of size %u into file %s \n", (U32)dictSize, dictFileName); @@ -122,7 +122,7 @@ int RANDOM_trainFromFiles(const char* dictFileName, sampleInfo *info, } /* clean up */ -_cleanup: +_done: free(dictBuffer); return result; } From 0c5eaef248443342dd1cd19f5e434334bef6fc4c Mon Sep 17 00:00:00 2001 From: Jennifer Liu Date: Thu, 19 Jul 2018 13:44:27 -0700 Subject: [PATCH 067/372] Update Makefile --- contrib/randomDictBuilder/Makefile | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/contrib/randomDictBuilder/Makefile b/contrib/randomDictBuilder/Makefile index 678ff28a8..5f9240bf6 100644 --- a/contrib/randomDictBuilder/Makefile +++ b/contrib/randomDictBuilder/Makefile @@ -1,8 +1,11 @@ -PROGRAM_FILES := ../../programs/fileio.c +ARG := + +CC ?= gcc +CFLAGS ?= -O3 +INCLUDES := -I ../../programs -I ../../lib/common -I ../../lib -I ../../lib/dictBuilder TEST_INPUT := ../../lib TEST_OUTPUT := randomDict -ARG := all: main run clean @@ -15,16 +18,16 @@ run: ./main $(ARG) main: main.o io.o random.o libzstd.a - gcc main.o io.o random.o libzstd.a -o main + $(CC) $(CFLAGS) main.o io.o random.o libzstd.a -o main main.o: main.c - gcc -c main.c -I io.h -I random.h -I ../../programs -I ../../lib/common -I ../../lib -I ../../lib/dictBuilder + $(CC) $(CFLAGS) $(INCLUDES) -c main.c random.o: random.c - gcc -c random.c -I random.h -I ../../lib/common -I ../../lib/dictBuilder + $(CC) $(CFLAGS) $(INCLUDES) -c random.c -io.o: io.c $(PROGRAM_FILES) - gcc -c io.c $(PROGRAM_FILES) -I io.h -I ../../programs -I ../../lib/common -I ../../lib -I ../../lib/dictBuilder +io.o: io.c + $(CC) $(CFLAGS) $(INCLUDES) -c io.c libzstd.a: $(MAKE) -C ../../lib libzstd.a @@ -44,8 +47,6 @@ testshell: test.sh .PHONY: clean clean: - rm -f libzstd.a main - rm -f ../../lib/*/*.o - rm -f ../../programs/*.o - rm -f *.o + rm -f *.o main libzstd.a + $(MAKE) -C ../../lib clean echo "Cleaning is completed" From 5624f3f1eabf84d603ca5607f59e6aa286d13211 Mon Sep 17 00:00:00 2001 From: Nick Terrell Date: Thu, 19 Jul 2018 14:35:27 -0700 Subject: [PATCH 068/372] Revert "attempt to re-enable arm64 tests" This reverts commit 9c277f137cbcaa385ff5b95ec4cbdce50675541d. --- .travis.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 80406064d..71b270196 100644 --- a/.travis.yml +++ b/.travis.yml @@ -25,7 +25,12 @@ matrix: - env: Cmd='make valgrindinstall && make -C tests clean valgrindTest' - env: Cmd='make arminstall && make armfuzz' - - env: Cmd='make arminstall && make aarch64fuzz' + +# Following test is disabled, as there is a bug in Travis' ld +# preventing aarch64 compilation to complete. +# > collect2: error: ld terminated with signal 11 [Segmentation fault], core dumped +# to be re-enabled in a few commit, as it's possible that a random code change circumvent the ld bug +# - env: Cmd='make arminstall && make aarch64fuzz' - env: Cmd='make ppcinstall && make ppcfuzz' - env: Cmd='make ppcinstall && make ppc64fuzz' From 470c8d42f4bbc8246bcd0bc8438aaad6d1c375ee Mon Sep 17 00:00:00 2001 From: Jennifer Liu Date: Fri, 20 Jul 2018 11:32:39 -0700 Subject: [PATCH 069/372] Benchmark dictionary builders --- contrib/benchmarkDictBuilder/Makefile | 44 ++ contrib/benchmarkDictBuilder/README.md | 43 ++ contrib/benchmarkDictBuilder/benchmark.c | 458 +++++++++++++++++++++ contrib/benchmarkDictBuilder/dictBuilder.h | 10 + contrib/benchmarkDictBuilder/test.sh | 2 + contrib/randomDictBuilder/io.c | 2 +- contrib/randomDictBuilder/io.h | 4 + 7 files changed, 562 insertions(+), 1 deletion(-) create mode 100644 contrib/benchmarkDictBuilder/Makefile create mode 100644 contrib/benchmarkDictBuilder/README.md create mode 100644 contrib/benchmarkDictBuilder/benchmark.c create mode 100644 contrib/benchmarkDictBuilder/dictBuilder.h create mode 100644 contrib/benchmarkDictBuilder/test.sh diff --git a/contrib/benchmarkDictBuilder/Makefile b/contrib/benchmarkDictBuilder/Makefile new file mode 100644 index 000000000..d36d96d52 --- /dev/null +++ b/contrib/benchmarkDictBuilder/Makefile @@ -0,0 +1,44 @@ +ARG := + +CC ?= gcc +CFLAGS ?= -O3 +INCLUDES := -I ../randomDictBuilder -I ../../programs -I ../../lib/common -I ../../lib -I ../../lib/dictBuilder + +RANDOM_FILE := ../randomDictBuilder/random.c +IO_FILE := ../randomDictBuilder/io.c + +all: run clean + +.PHONY: run +run: benchmark + echo "Benchmarking with $(ARG)" + ./benchmark $(ARG) + +.PHONY: test +test: benchmarkTest clean + +.PHONY: benchmarkTest +benchmarkTest: benchmark test.sh + sh test.sh + +benchmark: benchmark.o io.o random.o libzstd.a + $(CC) $(CFLAGS) benchmark.o io.o random.o libzstd.a -o benchmark + +benchmark.o: benchmark.c + $(CC) $(CFLAGS) $(INCLUDES) -c benchmark.c + +random.o: $(RANDOM_FILE) + $(CC) $(CFLAGS) $(INCLUDES) -c $(RANDOM_FILE) + +io.o: $(IO_FILE) + $(CC) $(CFLAGS) $(INCLUDES) -c $(IO_FILE) + +libzstd.a: + $(MAKE) -C ../../lib libzstd.a + mv ../../lib/libzstd.a . + +.PHONY: clean +clean: + rm -f *.o benchmark libzstd.a + $(MAKE) -C ../../lib clean + echo "Cleaning is completed" diff --git a/contrib/benchmarkDictBuilder/README.md b/contrib/benchmarkDictBuilder/README.md new file mode 100644 index 000000000..b680a53c6 --- /dev/null +++ b/contrib/benchmarkDictBuilder/README.md @@ -0,0 +1,43 @@ +Benchmarking Dictionary Builder + +### Permitted Argument: +Input File/Directory (in=fileName): required; file/directory used to build dictionary; if directory, will operate recursively for files inside directory; can include multiple files/directories, each following "in=" + +###Running Test: +make test + +###Usage: +Benchmark given input files: make ARG= followed by permitted arguments + +### Examples: +make ARG="in=../../lib/dictBuilder in=../../lib/compress" + +###Benchmarking Result: + +github: +| Algorithm | Speed(sec) | Compression Ratio | +| ------------- |:-------------:| ------------------:| +| random | 0.182254 | 8.786957 | +| cover | 34.821007 | 10.430999 | +| legacy | 1.125494 | 8.989482 | + +hg-commands +| Algorithm | Speed(sec) | Compression Ratio | +| ------------- |:-------------:| ------------------:| +| random | 0.089231 | 3.489515 | +| cover | 32.342462 | 4.030274 | +| legacy | 1.066594 | 3.911896 | + +hg-manifest +| Algorithm | Speed(sec) | Compression Ratio | +| ------------- |:-------------:| ------------------:| +| random | 1.095083 | 2.309485 | +| cover | 517.999132 | 2.575331 | +| legacy | 10.789509 | 2.506775 | + +hg-changelog +| Algorithm | Speed(sec) | Compression Ratio | +| ------------- |:-------------:| ------------------:| +| random | 0.639630 | 2.096785 | +| cover | 121.398023 | 2.175706 | +| legacy | 3.050893 | 2.058273 | diff --git a/contrib/benchmarkDictBuilder/benchmark.c b/contrib/benchmarkDictBuilder/benchmark.c new file mode 100644 index 000000000..aabd96a08 --- /dev/null +++ b/contrib/benchmarkDictBuilder/benchmark.c @@ -0,0 +1,458 @@ +#include /* fprintf */ +#include /* malloc, free, qsort */ +#include /* strcmp, strlen */ +#include /* errno */ +#include +#include +#include "random.h" +#include "dictBuilder.h" +#include "zstd_internal.h" /* includes zstd.h */ +#include "io.h" +#include "util.h" +#include "zdict.h" + + + +/*-************************************* +* Console display +***************************************/ +#define DISPLAY(...) fprintf(stderr, __VA_ARGS__) +#define DISPLAYLEVEL(l, ...) if (displayLevel>=l) { DISPLAY(__VA_ARGS__); } + +static const U64 g_refreshRate = SEC_TO_MICRO / 6; +static UTIL_time_t g_displayClock = UTIL_TIME_INITIALIZER; + +#define DISPLAYUPDATE(l, ...) { if (displayLevel>=l) { \ + if ((UTIL_clockSpanMicro(g_displayClock) > g_refreshRate) || (displayLevel>=4)) \ + { g_displayClock = UTIL_getTime(); DISPLAY(__VA_ARGS__); \ + if (displayLevel>=4) fflush(stderr); } } } + + +/*-************************************* +* Exceptions +***************************************/ +#ifndef DEBUG +# define DEBUG 0 +#endif +#define DEBUGOUTPUT(...) if (DEBUG) DISPLAY(__VA_ARGS__); +#define EXM_THROW(error, ...) \ +{ \ + DEBUGOUTPUT("Error defined at %s, line %i : \n", __FILE__, __LINE__); \ + DISPLAY("Error %i : ", error); \ + DISPLAY(__VA_ARGS__); \ + DISPLAY("\n"); \ + exit(error); \ +} + +/*-************************************* +* Constants +***************************************/ +static const unsigned g_defaultMaxDictSize = 110 KB; +#define MEMMULT 11 +#define NOISELENGTH 32 + +/*-************************************* +* Struct +***************************************/ +typedef struct { + const void* dictBuffer; + size_t dictSize; +} dictInfo; + + +/*-************************************* +* Commandline related functions +***************************************/ +static unsigned readU32FromChar(const char** stringPtr){ + const char errorMsg[] = "error: numeric value too large"; + unsigned result = 0; + while ((**stringPtr >='0') && (**stringPtr <='9')) { + unsigned const max = (((unsigned)(-1)) / 10) - 1; + if (result > max) exit(1); + result *= 10, result += **stringPtr - '0', (*stringPtr)++ ; + } + if ((**stringPtr=='K') || (**stringPtr=='M')) { + unsigned const maxK = ((unsigned)(-1)) >> 10; + if (result > maxK) exit(1); + result <<= 10; + if (**stringPtr=='M') { + if (result > maxK) exit(1); + result <<= 10; + } + (*stringPtr)++; /* skip `K` or `M` */ + if (**stringPtr=='i') (*stringPtr)++; + if (**stringPtr=='B') (*stringPtr)++; + } + return result; +} + +/** longCommandWArg() : + * check if *stringPtr is the same as longCommand. + * If yes, @return 1 and advances *stringPtr to the position which immediately follows longCommand. + * @return 0 and doesn't modify *stringPtr otherwise. + */ +static unsigned longCommandWArg(const char** stringPtr, const char* longCommand){ + size_t const comSize = strlen(longCommand); + int const result = !strncmp(*stringPtr, longCommand, comSize); + if (result) *stringPtr += comSize; + return result; +} + +static void fillNoise(void* buffer, size_t length) +{ + unsigned const prime1 = 2654435761U; + unsigned const prime2 = 2246822519U; + unsigned acc = prime1; + size_t p=0;; + + for (p=0; p> 21); + } +} + +/*-************************************* +* Dictionary related operations +***************************************/ +/** createDictFromFiles() : + * Based on type of param given, train dictionary using the corresponding algorithm + * @return dictInfo containing dictionary buffer and dictionary size + */ +dictInfo* createDictFromFiles(sampleInfo *info, unsigned maxDictSize, + ZDICT_random_params_t *randomParams, ZDICT_cover_params_t *coverParams, + ZDICT_legacy_params_t *legacyParams) { + unsigned const displayLevel = randomParams ? randomParams->zParams.notificationLevel : + coverParams ? coverParams->zParams.notificationLevel : + legacyParams ? legacyParams->zParams.notificationLevel : + 0; /* should never happen */ + void* const dictBuffer = malloc(maxDictSize); + + dictInfo* dInfo; + + /* Checks */ + if (!dictBuffer) + EXM_THROW(12, "not enough memory for trainFromFiles"); /* should not happen */ + + { size_t dictSize; + if(randomParams) { + dictSize = ZDICT_trainFromBuffer_random(dictBuffer, maxDictSize, info->srcBuffer, + info->samplesSizes, info->nbSamples, *randomParams); + }else if(coverParams) { + dictSize = ZDICT_optimizeTrainFromBuffer_cover(dictBuffer, maxDictSize, info->srcBuffer, + info->samplesSizes, info->nbSamples, coverParams); + } else { + size_t totalSize= 0; + for (int i = 0; i < info->nbSamples; i++) { + totalSize += info->samplesSizes[i]; + } + size_t const maxMem = findMaxMem(totalSize * MEMMULT) / MEMMULT; + size_t loadedSize = (size_t) MIN ((unsigned long long)maxMem, totalSize); + fillNoise((char*)(info->srcBuffer) + loadedSize, NOISELENGTH); + dictSize = ZDICT_trainFromBuffer_unsafe_legacy(dictBuffer, maxDictSize, info->srcBuffer, + info->samplesSizes, info->nbSamples, *legacyParams); + } + if (ZDICT_isError(dictSize)) { + DISPLAYLEVEL(1, "dictionary training failed : %s \n", ZDICT_getErrorName(dictSize)); /* should not happen */ + free(dictBuffer); + freeSampleInfo(info); + return dInfo; + } + dInfo = (dictInfo *)malloc(sizeof(dictInfo)); + dInfo->dictBuffer = dictBuffer; + dInfo->dictSize = dictSize; + } + return dInfo; +} + + +/** compressWithDict() : + * Compress samples from sample buffer given dicionary stored on dictionary buffer and compression level + * @return compression ratio + */ +double compressWithDict(sampleInfo *srcInfo, dictInfo* dInfo, int compressionLevel, int displayLevel) { + /* Local variables */ + size_t totalCompressedSize = 0; + size_t totalOriginalSize = 0; + double cRatio; + size_t dstCapacity; + int i; + + /* Pointers */ + ZSTD_CCtx* cctx; + ZSTD_CDict *cdict; + size_t *offsets; + void* dst; + + /* Allocate dst with enough space to compress the maximum sized sample */ + { + size_t maxSampleSize = 0; + for (int i = 0; i < srcInfo->nbSamples; i++) { + maxSampleSize = MAX(srcInfo->samplesSizes[i], maxSampleSize); + } + dstCapacity = ZSTD_compressBound(maxSampleSize); + dst = malloc(dstCapacity); + } + + /* Create the cctx and cdict */ + cctx = ZSTD_createCCtx(); + cdict = ZSTD_createCDict(dInfo->dictBuffer, dInfo->dictSize, compressionLevel); + + if(!cctx || !cdict || !dst) { + cRatio = -1; + goto _cleanup; + } + + /* Calculate offset for each sample */ + offsets = (size_t *)malloc((srcInfo->nbSamples + 1) * sizeof(size_t)); + offsets[0] = 0; + for (i = 1; i <= srcInfo->nbSamples; i++) { + offsets[i] = offsets[i - 1] + srcInfo->samplesSizes[i - 1]; + } + + /* Compress each sample and sum their sizes*/ + const BYTE *const samples = (const BYTE *)srcInfo->srcBuffer; + for (i = 0; i < srcInfo->nbSamples; i++) { + const size_t compressedSize = ZSTD_compress_usingCDict(cctx, dst, dstCapacity, samples + offsets[i], srcInfo->samplesSizes[i], cdict); + if (ZSTD_isError(compressedSize)) { + cRatio = -1; + goto _cleanup; + } + totalCompressedSize += compressedSize; + } + + /* Sum orignal sizes */ + for (i = 0; inbSamples; i++) { + totalOriginalSize += srcInfo->samplesSizes[i]; + } + + /* Calculate compression ratio */ + DISPLAYLEVEL(2, "original size is %lu\n", totalOriginalSize); + DISPLAYLEVEL(2, "compressed size is %lu\n", totalCompressedSize); + cRatio = (double)totalOriginalSize/(double)totalCompressedSize; + +_cleanup: + if(dst) { + free(dst); + } + if(offsets) { + free(offsets); + } + ZSTD_freeCCtx(cctx); + ZSTD_freeCDict(cdict); + return cRatio; +} + + +/** FreeDictInfo() : + * Free memory allocated for dictInfo + */ +void freeDictInfo(dictInfo* info) { + if (!info) return; + if (info->dictBuffer) free((void*)(info->dictBuffer)); + free(info); +} + + + +/*-******************************************************** + * Benchmarking functions +**********************************************************/ +/** benchmarkRandom() : + * Measure how long random dictionary builder takes and compression ratio with the random dictionary + * @return 0 if benchmark successfully, 1 otherwise + */ +int benchmarkRandom(sampleInfo *srcInfo, unsigned maxDictSize, ZDICT_random_params_t *randomParam) { + const int displayLevel = randomParam->zParams.notificationLevel; + int result = 0; + clock_t t; + t = clock(); + dictInfo* dInfo = createDictFromFiles(srcInfo, maxDictSize, randomParam, NULL, NULL); + t = clock() - t; + double time_taken = ((double)t)/CLOCKS_PER_SEC; + if (!dInfo) { + DISPLAYLEVEL(1, "RANDOM does not train successfully\n"); + result = 1; + goto _cleanup; + } + DISPLAYLEVEL(2, "RANDOM took %f seconds to execute \n", time_taken); + + double cRatio = compressWithDict(srcInfo, dInfo, randomParam->zParams.compressionLevel, displayLevel); + if (cRatio < 0) { + DISPLAYLEVEL(1, "Compressing with RANDOM dictionary does not work\n"); + result = 1; + goto _cleanup; + } + DISPLAYLEVEL(2, "Compression ratio with random dictionary is %f\n", cRatio); + + +_cleanup: + freeDictInfo(dInfo); + return result; +} + +/** benchmarkCover() : + * Measure how long random dictionary builder takes and compression ratio with the cover dictionary + * @return 0 if benchmark successfully, 1 otherwise + */ +int benchmarkCover(sampleInfo *srcInfo, unsigned maxDictSize, + ZDICT_cover_params_t *coverParam) { + const int displayLevel = coverParam->zParams.notificationLevel; + int result = 0; + clock_t t; + t = clock(); + dictInfo* dInfo = createDictFromFiles(srcInfo, maxDictSize, NULL, coverParam, NULL); + t = clock() - t; + double time_taken = ((double)t)/CLOCKS_PER_SEC; + if (!dInfo) { + DISPLAYLEVEL(1, "COVER does not train successfully\n"); + result = 1; + goto _cleanup; + } + DISPLAYLEVEL(2, "COVER took %f seconds to execute \n", time_taken); + + double cRatio = compressWithDict(srcInfo, dInfo, coverParam->zParams.compressionLevel, displayLevel); + if (cRatio < 0) { + DISPLAYLEVEL(1, "Compressing with COVER dictionary does not work\n"); + result = 1; + goto _cleanup; + } + DISPLAYLEVEL(2, "Compression ratio with cover dictionary is %f\n", cRatio); + +_cleanup: + freeDictInfo(dInfo); + return result; +} + + + +/** benchmarkLegacy() : + * Measure how long legacy dictionary builder takes and compression ratio with the legacy dictionary + * @return 0 if benchmark successfully, 1 otherwise + */ +int benchmarkLegacy(sampleInfo *srcInfo, unsigned maxDictSize, ZDICT_legacy_params_t *legacyParam) { + const int displayLevel = legacyParam->zParams.notificationLevel; + int result = 0; + clock_t t; + t = clock(); + dictInfo* dInfo = createDictFromFiles(srcInfo, maxDictSize, NULL, NULL, legacyParam); + t = clock() - t; + double time_taken = ((double)t)/CLOCKS_PER_SEC; + if (!dInfo) { + DISPLAYLEVEL(1, "LEGACY does not train successfully\n"); + result = 1; + goto _cleanup; + + } + DISPLAYLEVEL(2, "LEGACY took %f seconds to execute \n", time_taken); + + double cRatio = compressWithDict(srcInfo, dInfo, legacyParam->zParams.compressionLevel, displayLevel); + if (cRatio < 0) { + DISPLAYLEVEL(1, "Compressing with LEGACY dictionary does not work\n"); + result = 1; + goto _cleanup; + + } + DISPLAYLEVEL(2, "Compression ratio with legacy dictionary is %f\n", cRatio); + +_cleanup: + freeDictInfo(dInfo); + return result; +} + + + +int main(int argCount, const char* argv[]) +{ + int displayLevel = 2; + const char* programName = argv[0]; + int result = 0; + /* Initialize arguments to default values */ + unsigned k = 200; + unsigned d = 6; + unsigned cLevel = 3; + unsigned dictID = 0; + unsigned maxDictSize = g_defaultMaxDictSize; + + /* Initialize table to store input files */ + const char** filenameTable = (const char**)malloc(argCount * sizeof(const char*)); + unsigned filenameIdx = 0; + + char* fileNamesBuf = NULL; + unsigned fileNamesNb = filenameIdx; + int followLinks = 0; + const char** extendedFileList = NULL; + + /* Parse arguments */ + for (int i = 1; i < argCount; i++) { + const char* argument = argv[i]; + if (longCommandWArg(&argument, "in=")) { + filenameTable[filenameIdx] = argument; + filenameIdx++; + continue; + } + DISPLAYLEVEL(1, "benchmark: Incorrect parameters\n"); + return 1; + } + + + /* Get the list of all files recursively (because followLinks==0)*/ + extendedFileList = UTIL_createFileList(filenameTable, filenameIdx, &fileNamesBuf, + &fileNamesNb, followLinks); + if (extendedFileList) { + unsigned u; + for (u=0; u Date: Fri, 20 Jul 2018 17:03:47 -0700 Subject: [PATCH 070/372] Refactoring and benchmark without dictionary --- contrib/benchmarkDictBuilder/README.md | 43 --- contrib/benchmarkDictBuilder/dictBuilder.h | 10 - .../benchmarkDictBuilder/Makefile | 8 +- .../benchmarkDictBuilder/README.md | 47 +++ .../benchmarkDictBuilder/benchmark.c | 322 +++++++----------- .../benchmarkDictBuilder/dictBuilder.h | 6 + .../benchmarkDictBuilder/test.sh | 2 +- .../randomDictBuilder/Makefile | 10 +- .../randomDictBuilder/README.md | 4 +- .../randomDictBuilder/io.c | 33 ++ .../randomDictBuilder/io.h | 8 +- .../randomDictBuilder/main.c | 40 --- .../randomDictBuilder/random.c | 0 .../randomDictBuilder/random.h | 0 .../randomDictBuilder/test.sh | 12 +- 15 files changed, 232 insertions(+), 313 deletions(-) delete mode 100644 contrib/benchmarkDictBuilder/README.md delete mode 100644 contrib/benchmarkDictBuilder/dictBuilder.h rename contrib/{ => experimental_dict_builders}/benchmarkDictBuilder/Makefile (76%) create mode 100644 contrib/experimental_dict_builders/benchmarkDictBuilder/README.md rename contrib/{ => experimental_dict_builders}/benchmarkDictBuilder/benchmark.c (53%) create mode 100644 contrib/experimental_dict_builders/benchmarkDictBuilder/dictBuilder.h rename contrib/{ => experimental_dict_builders}/benchmarkDictBuilder/test.sh (54%) rename contrib/{ => experimental_dict_builders}/randomDictBuilder/Makefile (79%) rename contrib/{ => experimental_dict_builders}/randomDictBuilder/README.md (85%) rename contrib/{ => experimental_dict_builders}/randomDictBuilder/io.c (89%) rename contrib/{ => experimental_dict_builders}/randomDictBuilder/io.h (78%) rename contrib/{ => experimental_dict_builders}/randomDictBuilder/main.c (79%) rename contrib/{ => experimental_dict_builders}/randomDictBuilder/random.c (100%) rename contrib/{ => experimental_dict_builders}/randomDictBuilder/random.h (100%) rename contrib/{ => experimental_dict_builders}/randomDictBuilder/test.sh (52%) diff --git a/contrib/benchmarkDictBuilder/README.md b/contrib/benchmarkDictBuilder/README.md deleted file mode 100644 index b680a53c6..000000000 --- a/contrib/benchmarkDictBuilder/README.md +++ /dev/null @@ -1,43 +0,0 @@ -Benchmarking Dictionary Builder - -### Permitted Argument: -Input File/Directory (in=fileName): required; file/directory used to build dictionary; if directory, will operate recursively for files inside directory; can include multiple files/directories, each following "in=" - -###Running Test: -make test - -###Usage: -Benchmark given input files: make ARG= followed by permitted arguments - -### Examples: -make ARG="in=../../lib/dictBuilder in=../../lib/compress" - -###Benchmarking Result: - -github: -| Algorithm | Speed(sec) | Compression Ratio | -| ------------- |:-------------:| ------------------:| -| random | 0.182254 | 8.786957 | -| cover | 34.821007 | 10.430999 | -| legacy | 1.125494 | 8.989482 | - -hg-commands -| Algorithm | Speed(sec) | Compression Ratio | -| ------------- |:-------------:| ------------------:| -| random | 0.089231 | 3.489515 | -| cover | 32.342462 | 4.030274 | -| legacy | 1.066594 | 3.911896 | - -hg-manifest -| Algorithm | Speed(sec) | Compression Ratio | -| ------------- |:-------------:| ------------------:| -| random | 1.095083 | 2.309485 | -| cover | 517.999132 | 2.575331 | -| legacy | 10.789509 | 2.506775 | - -hg-changelog -| Algorithm | Speed(sec) | Compression Ratio | -| ------------- |:-------------:| ------------------:| -| random | 0.639630 | 2.096785 | -| cover | 121.398023 | 2.175706 | -| legacy | 3.050893 | 2.058273 | diff --git a/contrib/benchmarkDictBuilder/dictBuilder.h b/contrib/benchmarkDictBuilder/dictBuilder.h deleted file mode 100644 index a2dae5768..000000000 --- a/contrib/benchmarkDictBuilder/dictBuilder.h +++ /dev/null @@ -1,10 +0,0 @@ -/*! ZDICT_trainFromBuffer_unsafe_legacy() : - Strictly Internal use only !! - Same as ZDICT_trainFromBuffer_legacy(), but does not control `samplesBuffer`. - `samplesBuffer` must be followed by noisy guard band to avoid out-of-buffer reads. - @return : size of dictionary stored into `dictBuffer` (<= `dictBufferCapacity`) - or an error code. -*/ -size_t ZDICT_trainFromBuffer_unsafe_legacy(void* dictBuffer, size_t dictBufferCapacity, - const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples, - ZDICT_legacy_params_t parameters); diff --git a/contrib/benchmarkDictBuilder/Makefile b/contrib/experimental_dict_builders/benchmarkDictBuilder/Makefile similarity index 76% rename from contrib/benchmarkDictBuilder/Makefile rename to contrib/experimental_dict_builders/benchmarkDictBuilder/Makefile index d36d96d52..72ce04f2a 100644 --- a/contrib/benchmarkDictBuilder/Makefile +++ b/contrib/experimental_dict_builders/benchmarkDictBuilder/Makefile @@ -2,7 +2,7 @@ ARG := CC ?= gcc CFLAGS ?= -O3 -INCLUDES := -I ../randomDictBuilder -I ../../programs -I ../../lib/common -I ../../lib -I ../../lib/dictBuilder +INCLUDES := -I ../randomDictBuilder -I ../../../programs -I ../../../lib/common -I ../../../lib -I ../../../lib/dictBuilder RANDOM_FILE := ../randomDictBuilder/random.c IO_FILE := ../randomDictBuilder/io.c @@ -34,11 +34,11 @@ io.o: $(IO_FILE) $(CC) $(CFLAGS) $(INCLUDES) -c $(IO_FILE) libzstd.a: - $(MAKE) -C ../../lib libzstd.a - mv ../../lib/libzstd.a . + $(MAKE) -C ../../../lib libzstd.a + mv ../../../lib/libzstd.a . .PHONY: clean clean: rm -f *.o benchmark libzstd.a - $(MAKE) -C ../../lib clean + $(MAKE) -C ../../../lib clean echo "Cleaning is completed" diff --git a/contrib/experimental_dict_builders/benchmarkDictBuilder/README.md b/contrib/experimental_dict_builders/benchmarkDictBuilder/README.md new file mode 100644 index 000000000..de783a0ec --- /dev/null +++ b/contrib/experimental_dict_builders/benchmarkDictBuilder/README.md @@ -0,0 +1,47 @@ +Benchmarking Dictionary Builder + +### Permitted Argument: +Input File/Directory (in=fileName): required; file/directory used to build dictionary; if directory, will operate recursively for files inside directory; can include multiple files/directories, each following "in=" + +###Running Test: +make test + +###Usage: +Benchmark given input files: make ARG= followed by permitted arguments + +### Examples: +make ARG="in=../../../lib/dictBuilder in=../../../lib/compress" + +###Benchmarking Result: + +github: +| Algorithm | Speed(sec) | Compression Ratio | +| ------------- |:-------------:| ------------------:| +| nodict | 0.000004 | 2.999642 | +| random | 0.180238 | 8.786957 | +| cover | 33.891987 | 10.430999 | +| legacy | 1.077569 | 8.989482 | + +hg-commands +| Algorithm | Speed(sec) | Compression Ratio | +| ------------- |:-------------:| ------------------:| +| nodict | 0.000006 | 2.425291 | +| random | 0.088735 | 3.489515 | +| cover | 35.447300 | 4.030274 | +| legacy | 1.048509 | 3.911896 | + +hg-manifest +| Algorithm | Speed(sec) | Compression Ratio | +| ------------- |:-------------:| ------------------:| +| nodict | 0.000005 | 1.866385 | +| random | 1.148231 | 2.309485 | +| cover | 509.685257 | 2.575331 | +| legacy | 10.705866 | 2.506775 | + +hg-changelog +| Algorithm | Speed(sec) | Compression Ratio | +| ------------- |:-------------:| ------------------:| +| nodict | 0.000005 | 1.377613 | +| random | 0.706434 | 2.096785 | +| cover | 122.815783 | 2.175706 | +| legacy | 3.010318 | 2.058273 | diff --git a/contrib/benchmarkDictBuilder/benchmark.c b/contrib/experimental_dict_builders/benchmarkDictBuilder/benchmark.c similarity index 53% rename from contrib/benchmarkDictBuilder/benchmark.c rename to contrib/experimental_dict_builders/benchmarkDictBuilder/benchmark.c index aabd96a08..890afb8b4 100644 --- a/contrib/benchmarkDictBuilder/benchmark.c +++ b/contrib/experimental_dict_builders/benchmarkDictBuilder/benchmark.c @@ -44,12 +44,14 @@ static UTIL_time_t g_displayClock = UTIL_TIME_INITIALIZER; exit(error); \ } + /*-************************************* * Constants ***************************************/ static const unsigned g_defaultMaxDictSize = 110 KB; -#define MEMMULT 11 -#define NOISELENGTH 32 +#define DEFAULT_CLEVEL 3 +#define DEFAULT_DISPLAYLEVEL 2 + /*-************************************* * Struct @@ -60,57 +62,6 @@ typedef struct { } dictInfo; -/*-************************************* -* Commandline related functions -***************************************/ -static unsigned readU32FromChar(const char** stringPtr){ - const char errorMsg[] = "error: numeric value too large"; - unsigned result = 0; - while ((**stringPtr >='0') && (**stringPtr <='9')) { - unsigned const max = (((unsigned)(-1)) / 10) - 1; - if (result > max) exit(1); - result *= 10, result += **stringPtr - '0', (*stringPtr)++ ; - } - if ((**stringPtr=='K') || (**stringPtr=='M')) { - unsigned const maxK = ((unsigned)(-1)) >> 10; - if (result > maxK) exit(1); - result <<= 10; - if (**stringPtr=='M') { - if (result > maxK) exit(1); - result <<= 10; - } - (*stringPtr)++; /* skip `K` or `M` */ - if (**stringPtr=='i') (*stringPtr)++; - if (**stringPtr=='B') (*stringPtr)++; - } - return result; -} - -/** longCommandWArg() : - * check if *stringPtr is the same as longCommand. - * If yes, @return 1 and advances *stringPtr to the position which immediately follows longCommand. - * @return 0 and doesn't modify *stringPtr otherwise. - */ -static unsigned longCommandWArg(const char** stringPtr, const char* longCommand){ - size_t const comSize = strlen(longCommand); - int const result = !strncmp(*stringPtr, longCommand, comSize); - if (result) *stringPtr += comSize; - return result; -} - -static void fillNoise(void* buffer, size_t length) -{ - unsigned const prime1 = 2654435761U; - unsigned const prime2 = 2246822519U; - unsigned acc = prime1; - size_t p=0;; - - for (p=0; p> 21); - } -} - /*-************************************* * Dictionary related operations ***************************************/ @@ -122,9 +73,9 @@ dictInfo* createDictFromFiles(sampleInfo *info, unsigned maxDictSize, ZDICT_random_params_t *randomParams, ZDICT_cover_params_t *coverParams, ZDICT_legacy_params_t *legacyParams) { unsigned const displayLevel = randomParams ? randomParams->zParams.notificationLevel : - coverParams ? coverParams->zParams.notificationLevel : - legacyParams ? legacyParams->zParams.notificationLevel : - 0; /* should never happen */ + coverParams ? coverParams->zParams.notificationLevel : + legacyParams ? legacyParams->zParams.notificationLevel : + DEFAULT_DISPLAYLEVEL; /* no dict */ void* const dictBuffer = malloc(maxDictSize); dictInfo* dInfo; @@ -140,21 +91,15 @@ dictInfo* createDictFromFiles(sampleInfo *info, unsigned maxDictSize, }else if(coverParams) { dictSize = ZDICT_optimizeTrainFromBuffer_cover(dictBuffer, maxDictSize, info->srcBuffer, info->samplesSizes, info->nbSamples, coverParams); - } else { - size_t totalSize= 0; - for (int i = 0; i < info->nbSamples; i++) { - totalSize += info->samplesSizes[i]; - } - size_t const maxMem = findMaxMem(totalSize * MEMMULT) / MEMMULT; - size_t loadedSize = (size_t) MIN ((unsigned long long)maxMem, totalSize); - fillNoise((char*)(info->srcBuffer) + loadedSize, NOISELENGTH); - dictSize = ZDICT_trainFromBuffer_unsafe_legacy(dictBuffer, maxDictSize, info->srcBuffer, + } else if(legacyParams) { + dictSize = ZDICT_trainFromBuffer_legacy(dictBuffer, maxDictSize, info->srcBuffer, info->samplesSizes, info->nbSamples, *legacyParams); + } else { + dictSize = 0; } if (ZDICT_isError(dictSize)) { DISPLAYLEVEL(1, "dictionary training failed : %s \n", ZDICT_getErrorName(dictSize)); /* should not happen */ free(dictBuffer); - freeSampleInfo(info); return dInfo; } dInfo = (dictInfo *)malloc(sizeof(dictInfo)); @@ -173,6 +118,7 @@ double compressWithDict(sampleInfo *srcInfo, dictInfo* dInfo, int compressionLev /* Local variables */ size_t totalCompressedSize = 0; size_t totalOriginalSize = 0; + unsigned hasDict = dInfo->dictSize > 0 ? 1 : 0; double cRatio; size_t dstCapacity; int i; @@ -193,15 +139,6 @@ double compressWithDict(sampleInfo *srcInfo, dictInfo* dInfo, int compressionLev dst = malloc(dstCapacity); } - /* Create the cctx and cdict */ - cctx = ZSTD_createCCtx(); - cdict = ZSTD_createCDict(dInfo->dictBuffer, dInfo->dictSize, compressionLevel); - - if(!cctx || !cdict || !dst) { - cRatio = -1; - goto _cleanup; - } - /* Calculate offset for each sample */ offsets = (size_t *)malloc((srcInfo->nbSamples + 1) * sizeof(size_t)); offsets[0] = 0; @@ -209,13 +146,35 @@ double compressWithDict(sampleInfo *srcInfo, dictInfo* dInfo, int compressionLev offsets[i] = offsets[i - 1] + srcInfo->samplesSizes[i - 1]; } + /* Create the cctx */ + cctx = ZSTD_createCCtx(); + if(!cctx || !dst) { + cRatio = -1; + goto _nodictCleanup; + } + + /* Create CDict if there's a dictionary stored on buffer */ + if (hasDict) { + cdict = ZSTD_createCDict(dInfo->dictBuffer, dInfo->dictSize, compressionLevel); + if(!cdict) { + cRatio = -1; + goto _dictCleanup; + } + } + /* Compress each sample and sum their sizes*/ const BYTE *const samples = (const BYTE *)srcInfo->srcBuffer; for (i = 0; i < srcInfo->nbSamples; i++) { - const size_t compressedSize = ZSTD_compress_usingCDict(cctx, dst, dstCapacity, samples + offsets[i], srcInfo->samplesSizes[i], cdict); + size_t compressedSize; + if(hasDict) { + compressedSize = ZSTD_compress_usingCDict(cctx, dst, dstCapacity, samples + offsets[i], srcInfo->samplesSizes[i], cdict); + } else { + compressedSize = ZSTD_compressCCtx(cctx, dst, dstCapacity,samples + offsets[i], srcInfo->samplesSizes[i], compressionLevel); + } if (ZSTD_isError(compressedSize)) { cRatio = -1; - goto _cleanup; + if(hasDict) goto _dictCleanup; + else goto _nodictCleanup; } totalCompressedSize += compressedSize; } @@ -230,15 +189,14 @@ double compressWithDict(sampleInfo *srcInfo, dictInfo* dInfo, int compressionLev DISPLAYLEVEL(2, "compressed size is %lu\n", totalCompressedSize); cRatio = (double)totalOriginalSize/(double)totalCompressedSize; -_cleanup: - if(dst) { - free(dst); - } - if(offsets) { - free(offsets); - } - ZSTD_freeCCtx(cctx); +_dictCleanup: ZSTD_freeCDict(cdict); + +_nodictCleanup: + free(dst); + free(offsets); + ZSTD_freeCCtx(cctx); + return cRatio; } @@ -257,102 +215,48 @@ void freeDictInfo(dictInfo* info) { /*-******************************************************** * Benchmarking functions **********************************************************/ -/** benchmarkRandom() : - * Measure how long random dictionary builder takes and compression ratio with the random dictionary +/** benchmarkDictBuilder() : + * Measure how long a dictionary builder takes and compression ratio with the dictionary built * @return 0 if benchmark successfully, 1 otherwise */ -int benchmarkRandom(sampleInfo *srcInfo, unsigned maxDictSize, ZDICT_random_params_t *randomParam) { - const int displayLevel = randomParam->zParams.notificationLevel; +int benchmarkDictBuilder(sampleInfo *srcInfo, unsigned maxDictSize, ZDICT_random_params_t *randomParam, + ZDICT_cover_params_t *coverParam, ZDICT_legacy_params_t *legacyParam) { + /* Local variables */ + const unsigned displayLevel = randomParam ? randomParam->zParams.notificationLevel : + coverParam ? coverParam->zParams.notificationLevel : + legacyParam ? legacyParam->zParams.notificationLevel : + DEFAULT_DISPLAYLEVEL; /* no dict */ + const char* name = randomParam ? "RANDOM" : + coverParam ? "COVER" : + legacyParam ? "LEGACY" : + "NODICT"; /* no dict */ + const unsigned cLevel = randomParam ? randomParam->zParams.compressionLevel : + coverParam ? coverParam->zParams.compressionLevel : + legacyParam ? legacyParam->zParams.compressionLevel : + DEFAULT_CLEVEL; /* no dict */ int result = 0; - clock_t t; - t = clock(); - dictInfo* dInfo = createDictFromFiles(srcInfo, maxDictSize, randomParam, NULL, NULL); - t = clock() - t; - double time_taken = ((double)t)/CLOCKS_PER_SEC; + + /* Calculate speed */ + const UTIL_time_t begin = UTIL_getTime(); + dictInfo* dInfo = createDictFromFiles(srcInfo, maxDictSize, randomParam, coverParam, legacyParam); + const U64 timeMicro = UTIL_clockSpanMicro(begin); + const double timeSec = timeMicro / (double)SEC_TO_MICRO; if (!dInfo) { - DISPLAYLEVEL(1, "RANDOM does not train successfully\n"); + DISPLAYLEVEL(1, "%s does not train successfully\n", name); result = 1; goto _cleanup; } - DISPLAYLEVEL(2, "RANDOM took %f seconds to execute \n", time_taken); + DISPLAYLEVEL(2, "%s took %f seconds to execute \n", name, timeSec); - double cRatio = compressWithDict(srcInfo, dInfo, randomParam->zParams.compressionLevel, displayLevel); + /* Calculate compression ratio */ + double cRatio = compressWithDict(srcInfo, dInfo, cLevel, displayLevel); if (cRatio < 0) { - DISPLAYLEVEL(1, "Compressing with RANDOM dictionary does not work\n"); - result = 1; - goto _cleanup; - } - DISPLAYLEVEL(2, "Compression ratio with random dictionary is %f\n", cRatio); - - -_cleanup: - freeDictInfo(dInfo); - return result; -} - -/** benchmarkCover() : - * Measure how long random dictionary builder takes and compression ratio with the cover dictionary - * @return 0 if benchmark successfully, 1 otherwise - */ -int benchmarkCover(sampleInfo *srcInfo, unsigned maxDictSize, - ZDICT_cover_params_t *coverParam) { - const int displayLevel = coverParam->zParams.notificationLevel; - int result = 0; - clock_t t; - t = clock(); - dictInfo* dInfo = createDictFromFiles(srcInfo, maxDictSize, NULL, coverParam, NULL); - t = clock() - t; - double time_taken = ((double)t)/CLOCKS_PER_SEC; - if (!dInfo) { - DISPLAYLEVEL(1, "COVER does not train successfully\n"); - result = 1; - goto _cleanup; - } - DISPLAYLEVEL(2, "COVER took %f seconds to execute \n", time_taken); - - double cRatio = compressWithDict(srcInfo, dInfo, coverParam->zParams.compressionLevel, displayLevel); - if (cRatio < 0) { - DISPLAYLEVEL(1, "Compressing with COVER dictionary does not work\n"); - result = 1; - goto _cleanup; - } - DISPLAYLEVEL(2, "Compression ratio with cover dictionary is %f\n", cRatio); - -_cleanup: - freeDictInfo(dInfo); - return result; -} - - - -/** benchmarkLegacy() : - * Measure how long legacy dictionary builder takes and compression ratio with the legacy dictionary - * @return 0 if benchmark successfully, 1 otherwise - */ -int benchmarkLegacy(sampleInfo *srcInfo, unsigned maxDictSize, ZDICT_legacy_params_t *legacyParam) { - const int displayLevel = legacyParam->zParams.notificationLevel; - int result = 0; - clock_t t; - t = clock(); - dictInfo* dInfo = createDictFromFiles(srcInfo, maxDictSize, NULL, NULL, legacyParam); - t = clock() - t; - double time_taken = ((double)t)/CLOCKS_PER_SEC; - if (!dInfo) { - DISPLAYLEVEL(1, "LEGACY does not train successfully\n"); + DISPLAYLEVEL(1, "Compressing with %s dictionary does not work\n", name); result = 1; goto _cleanup; } - DISPLAYLEVEL(2, "LEGACY took %f seconds to execute \n", time_taken); - - double cRatio = compressWithDict(srcInfo, dInfo, legacyParam->zParams.compressionLevel, displayLevel); - if (cRatio < 0) { - DISPLAYLEVEL(1, "Compressing with LEGACY dictionary does not work\n"); - result = 1; - goto _cleanup; - - } - DISPLAYLEVEL(2, "Compression ratio with legacy dictionary is %f\n", cRatio); + DISPLAYLEVEL(2, "Compression ratio with %s dictionary is %f\n", name, cRatio); _cleanup: freeDictInfo(dInfo); @@ -363,15 +267,16 @@ _cleanup: int main(int argCount, const char* argv[]) { - int displayLevel = 2; + const int displayLevel = DEFAULT_DISPLAYLEVEL; const char* programName = argv[0]; int result = 0; + /* Initialize arguments to default values */ - unsigned k = 200; - unsigned d = 6; - unsigned cLevel = 3; - unsigned dictID = 0; - unsigned maxDictSize = g_defaultMaxDictSize; + const unsigned k = 200; + const unsigned d = 6; + const unsigned cLevel = DEFAULT_CLEVEL; + const unsigned dictID = 0; + const unsigned maxDictSize = g_defaultMaxDictSize; /* Initialize table to store input files */ const char** filenameTable = (const char**)malloc(argCount * sizeof(const char*)); @@ -379,7 +284,7 @@ int main(int argCount, const char* argv[]) char* fileNamesBuf = NULL; unsigned fileNamesNb = filenameIdx; - int followLinks = 0; + const int followLinks = 0; const char** extendedFileList = NULL; /* Parse arguments */ @@ -394,7 +299,6 @@ int main(int argCount, const char* argv[]) return 1; } - /* Get the list of all files recursively (because followLinks==0)*/ extendedFileList = UTIL_createFileList(filenameTable, filenameIdx, &fileNamesBuf, &fileNamesNb, followLinks); @@ -406,6 +310,7 @@ int main(int argCount, const char* argv[]) filenameIdx = fileNamesNb; } + /* get sampleInfo */ size_t blockSize = 0; sampleInfo* srcInfo= getSampleInfo(filenameTable, filenameIdx, blockSize, maxDictSize, displayLevel); @@ -416,38 +321,53 @@ int main(int argCount, const char* argv[]) zParams.notificationLevel = displayLevel; zParams.dictID = dictID; + /* with no dict */ + { + const int noDictResult = benchmarkDictBuilder(srcInfo, maxDictSize, NULL, NULL, NULL); + if(noDictResult) { + result = 1; + goto _cleanup; + } + } + /* for random */ - ZDICT_random_params_t randomParam; - randomParam.zParams = zParams; - randomParam.k = k; - int randomResult = benchmarkRandom(srcInfo, maxDictSize, &randomParam); - if(randomResult) { - result = 1; - goto _cleanup; + { + ZDICT_random_params_t randomParam; + randomParam.zParams = zParams; + randomParam.k = k; + const int randomResult = benchmarkDictBuilder(srcInfo, maxDictSize, &randomParam, NULL, NULL); + if(randomResult) { + result = 1; + goto _cleanup; + } } /* for cover */ - ZDICT_cover_params_t coverParam; - memset(&coverParam, 0, sizeof(coverParam)); - coverParam.zParams = zParams; - coverParam.splitPoint = 1.0; - coverParam.d = d; - coverParam.steps = 40; - coverParam.nbThreads = 1; - int coverOptResult = benchmarkCover(srcInfo, maxDictSize, &coverParam); - if(coverOptResult) { - result = 1; - goto _cleanup; + { + ZDICT_cover_params_t coverParam; + memset(&coverParam, 0, sizeof(coverParam)); + coverParam.zParams = zParams; + coverParam.splitPoint = 1.0; + coverParam.d = d; + coverParam.steps = 40; + coverParam.nbThreads = 1; + const int coverOptResult = benchmarkDictBuilder(srcInfo, maxDictSize, NULL, &coverParam, NULL); + if(coverOptResult) { + result = 1; + goto _cleanup; + } } /* for legacy */ - ZDICT_legacy_params_t legacyParam; - legacyParam.zParams = zParams; - legacyParam.selectivityLevel = 9; - int legacyResult = benchmarkLegacy(srcInfo, maxDictSize, &legacyParam); - if(legacyResult) { - result = 1; - goto _cleanup; + { + ZDICT_legacy_params_t legacyParam; + legacyParam.zParams = zParams; + legacyParam.selectivityLevel = 9; + const int legacyResult = benchmarkDictBuilder(srcInfo, maxDictSize, NULL, NULL, &legacyParam); + if(legacyResult) { + result = 1; + goto _cleanup; + } } /* Free allocated memory */ diff --git a/contrib/experimental_dict_builders/benchmarkDictBuilder/dictBuilder.h b/contrib/experimental_dict_builders/benchmarkDictBuilder/dictBuilder.h new file mode 100644 index 000000000..781ec8c2f --- /dev/null +++ b/contrib/experimental_dict_builders/benchmarkDictBuilder/dictBuilder.h @@ -0,0 +1,6 @@ +/* ZDICT_trainFromBuffer_legacy() : + * issue : samplesBuffer need to be followed by a noisy guard band. + * work around : duplicate the buffer, and add the noise */ +size_t ZDICT_trainFromBuffer_legacy(void* dictBuffer, size_t dictBufferCapacity, + const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples, + ZDICT_legacy_params_t params); diff --git a/contrib/benchmarkDictBuilder/test.sh b/contrib/experimental_dict_builders/benchmarkDictBuilder/test.sh similarity index 54% rename from contrib/benchmarkDictBuilder/test.sh rename to contrib/experimental_dict_builders/benchmarkDictBuilder/test.sh index 6354784e4..5eaf5930a 100644 --- a/contrib/benchmarkDictBuilder/test.sh +++ b/contrib/experimental_dict_builders/benchmarkDictBuilder/test.sh @@ -1,2 +1,2 @@ echo "Benchmark with in=../../lib/common" -./benchmark in=../../lib/common +./benchmark in=../../../lib/common diff --git a/contrib/randomDictBuilder/Makefile b/contrib/experimental_dict_builders/randomDictBuilder/Makefile similarity index 79% rename from contrib/randomDictBuilder/Makefile rename to contrib/experimental_dict_builders/randomDictBuilder/Makefile index 5f9240bf6..bbd40e47c 100644 --- a/contrib/randomDictBuilder/Makefile +++ b/contrib/experimental_dict_builders/randomDictBuilder/Makefile @@ -2,9 +2,9 @@ ARG := CC ?= gcc CFLAGS ?= -O3 -INCLUDES := -I ../../programs -I ../../lib/common -I ../../lib -I ../../lib/dictBuilder +INCLUDES := -I ../../../programs -I ../../../lib/common -I ../../../lib -I ../../../lib/dictBuilder -TEST_INPUT := ../../lib +TEST_INPUT := ../../../lib TEST_OUTPUT := randomDict all: main run clean @@ -30,8 +30,8 @@ io.o: io.c $(CC) $(CFLAGS) $(INCLUDES) -c io.c libzstd.a: - $(MAKE) -C ../../lib libzstd.a - mv ../../lib/libzstd.a . + $(MAKE) -C ../../../lib libzstd.a + mv ../../../lib/libzstd.a . .PHONY: testrun testrun: main @@ -48,5 +48,5 @@ testshell: test.sh .PHONY: clean clean: rm -f *.o main libzstd.a - $(MAKE) -C ../../lib clean + $(MAKE) -C ../../../lib clean echo "Cleaning is completed" diff --git a/contrib/randomDictBuilder/README.md b/contrib/experimental_dict_builders/randomDictBuilder/README.md similarity index 85% rename from contrib/randomDictBuilder/README.md rename to contrib/experimental_dict_builders/randomDictBuilder/README.md index 0e70d3dcc..da12a4280 100644 --- a/contrib/randomDictBuilder/README.md +++ b/contrib/experimental_dict_builders/randomDictBuilder/README.md @@ -16,5 +16,5 @@ To build a random dictionary with the provided arguments: make ARG= followed by ### Examples: -make ARG="in=../../lib/dictBuilder out=dict100 dictID=520" -make ARG="in=../../lib/dictBuilder in=../../lib/compress" +make ARG="in=../../../lib/dictBuilder out=dict100 dictID=520" +make ARG="in=../../../lib/dictBuilder in=../../../lib/compress" diff --git a/contrib/randomDictBuilder/io.c b/contrib/experimental_dict_builders/randomDictBuilder/io.c similarity index 89% rename from contrib/randomDictBuilder/io.c rename to contrib/experimental_dict_builders/randomDictBuilder/io.c index 1217b5747..bfe39eaed 100644 --- a/contrib/randomDictBuilder/io.c +++ b/contrib/experimental_dict_builders/randomDictBuilder/io.c @@ -53,6 +53,39 @@ static const size_t g_maxMemory = (sizeof(size_t) == 4) ? #define NOISELENGTH 32 +/*-************************************* +* Commandline related functions +***************************************/ +unsigned readU32FromChar(const char** stringPtr){ + const char errorMsg[] = "error: numeric value too large"; + unsigned result = 0; + while ((**stringPtr >='0') && (**stringPtr <='9')) { + unsigned const max = (((unsigned)(-1)) / 10) - 1; + if (result > max) exit(1); + result *= 10, result += **stringPtr - '0', (*stringPtr)++ ; + } + if ((**stringPtr=='K') || (**stringPtr=='M')) { + unsigned const maxK = ((unsigned)(-1)) >> 10; + if (result > maxK) exit(1); + result <<= 10; + if (**stringPtr=='M') { + if (result > maxK) exit(1); + result <<= 10; + } + (*stringPtr)++; /* skip `K` or `M` */ + if (**stringPtr=='i') (*stringPtr)++; + if (**stringPtr=='B') (*stringPtr)++; + } + return result; +} + +unsigned longCommandWArg(const char** stringPtr, const char* longCommand){ + size_t const comSize = strlen(longCommand); + int const result = !strncmp(*stringPtr, longCommand, comSize); + if (result) *stringPtr += comSize; + return result; +} + /* ******************************************************** * File related operations diff --git a/contrib/randomDictBuilder/io.h b/contrib/experimental_dict_builders/randomDictBuilder/io.h similarity index 78% rename from contrib/randomDictBuilder/io.h rename to contrib/experimental_dict_builders/randomDictBuilder/io.h index e2f454c26..0ee24604e 100644 --- a/contrib/randomDictBuilder/io.h +++ b/contrib/experimental_dict_builders/randomDictBuilder/io.h @@ -50,5 +50,11 @@ void freeSampleInfo(sampleInfo *info); void saveDict(const char* dictFileName, const void* buff, size_t buffSize); +unsigned readU32FromChar(const char** stringPtr); -size_t findMaxMem(unsigned long long requiredMem); +/** longCommandWArg() : + * check if *stringPtr is the same as longCommand. + * If yes, @return 1 and advances *stringPtr to the position which immediately follows longCommand. + * @return 0 and doesn't modify *stringPtr otherwise. + */ +unsigned longCommandWArg(const char** stringPtr, const char* longCommand); diff --git a/contrib/randomDictBuilder/main.c b/contrib/experimental_dict_builders/randomDictBuilder/main.c similarity index 79% rename from contrib/randomDictBuilder/main.c rename to contrib/experimental_dict_builders/randomDictBuilder/main.c index 4751a9e1c..3f3a6ca70 100644 --- a/contrib/randomDictBuilder/main.c +++ b/contrib/experimental_dict_builders/randomDictBuilder/main.c @@ -52,46 +52,6 @@ static const unsigned g_defaultMaxDictSize = 110 KB; -/*-************************************* -* Commandline related functions -***************************************/ -static unsigned readU32FromChar(const char** stringPtr){ - const char errorMsg[] = "error: numeric value too large"; - unsigned result = 0; - while ((**stringPtr >='0') && (**stringPtr <='9')) { - unsigned const max = (((unsigned)(-1)) / 10) - 1; - if (result > max) exit(1); - result *= 10, result += **stringPtr - '0', (*stringPtr)++ ; - } - if ((**stringPtr=='K') || (**stringPtr=='M')) { - unsigned const maxK = ((unsigned)(-1)) >> 10; - if (result > maxK) exit(1); - result <<= 10; - if (**stringPtr=='M') { - if (result > maxK) exit(1); - result <<= 10; - } - (*stringPtr)++; /* skip `K` or `M` */ - if (**stringPtr=='i') (*stringPtr)++; - if (**stringPtr=='B') (*stringPtr)++; - } - return result; -} - -/** longCommandWArg() : - * check if *stringPtr is the same as longCommand. - * If yes, @return 1 and advances *stringPtr to the position which immediately follows longCommand. - * @return 0 and doesn't modify *stringPtr otherwise. - */ -static unsigned longCommandWArg(const char** stringPtr, const char* longCommand){ - size_t const comSize = strlen(longCommand); - int const result = !strncmp(*stringPtr, longCommand, comSize); - if (result) *stringPtr += comSize; - return result; -} - - - /*-************************************* * RANDOM ***************************************/ diff --git a/contrib/randomDictBuilder/random.c b/contrib/experimental_dict_builders/randomDictBuilder/random.c similarity index 100% rename from contrib/randomDictBuilder/random.c rename to contrib/experimental_dict_builders/randomDictBuilder/random.c diff --git a/contrib/randomDictBuilder/random.h b/contrib/experimental_dict_builders/randomDictBuilder/random.h similarity index 100% rename from contrib/randomDictBuilder/random.h rename to contrib/experimental_dict_builders/randomDictBuilder/random.h diff --git a/contrib/randomDictBuilder/test.sh b/contrib/experimental_dict_builders/randomDictBuilder/test.sh similarity index 52% rename from contrib/randomDictBuilder/test.sh rename to contrib/experimental_dict_builders/randomDictBuilder/test.sh index 497820f88..1eb732e52 100644 --- a/contrib/randomDictBuilder/test.sh +++ b/contrib/experimental_dict_builders/randomDictBuilder/test.sh @@ -1,12 +1,12 @@ echo "Building random dictionary with in=../../lib/common k=200 out=dict1" -./main in=../../lib/common k=200 out=dict1 -zstd -be3 -D dict1 -r ../../lib/common -q +./main in=../../../lib/common k=200 out=dict1 +zstd -be3 -D dict1 -r ../../../lib/common -q echo "Building random dictionary with in=../../lib/common k=500 out=dict2 dictID=100 maxdict=140000" -./main in=../../lib/common k=500 out=dict2 dictID=100 maxdict=140000 -zstd -be3 -D dict2 -r ../../lib/common -q +./main in=../../../lib/common k=500 out=dict2 dictID=100 maxdict=140000 +zstd -be3 -D dict2 -r ../../../lib/common -q echo "Building random dictionary with 2 sample sources" -./main in=../../lib/common in=../../lib/compress out=dict3 -zstd -be3 -D dict3 -r ../../lib/common -q +./main in=../../../lib/common in=../../../lib/compress out=dict3 +zstd -be3 -D dict3 -r ../../../lib/common -q echo "Removing dict1 dict2 dict3" rm -f dict1 dict2 dict3 From b6c5d4982c489b76b4b0e994c680b1e3bd01080b Mon Sep 17 00:00:00 2001 From: Jennifer Liu Date: Fri, 20 Jul 2018 17:41:22 -0700 Subject: [PATCH 071/372] Minor fix --- .../benchmarkDictBuilder/benchmark.c | 28 ++++++++----------- 1 file changed, 12 insertions(+), 16 deletions(-) diff --git a/contrib/experimental_dict_builders/benchmarkDictBuilder/benchmark.c b/contrib/experimental_dict_builders/benchmarkDictBuilder/benchmark.c index 890afb8b4..640419649 100644 --- a/contrib/experimental_dict_builders/benchmarkDictBuilder/benchmark.c +++ b/contrib/experimental_dict_builders/benchmarkDictBuilder/benchmark.c @@ -78,7 +78,7 @@ dictInfo* createDictFromFiles(sampleInfo *info, unsigned maxDictSize, DEFAULT_DISPLAYLEVEL; /* no dict */ void* const dictBuffer = malloc(maxDictSize); - dictInfo* dInfo; + dictInfo* dInfo = NULL; /* Checks */ if (!dictBuffer) @@ -118,16 +118,16 @@ double compressWithDict(sampleInfo *srcInfo, dictInfo* dInfo, int compressionLev /* Local variables */ size_t totalCompressedSize = 0; size_t totalOriginalSize = 0; - unsigned hasDict = dInfo->dictSize > 0 ? 1 : 0; + const unsigned hasDict = dInfo->dictSize > 0 ? 1 : 0; double cRatio; size_t dstCapacity; int i; /* Pointers */ - ZSTD_CCtx* cctx; - ZSTD_CDict *cdict; - size_t *offsets; - void* dst; + ZSTD_CDict *cdict = NULL; + ZSTD_CCtx* cctx = NULL; + size_t *offsets = NULL; + void* dst = NULL; /* Allocate dst with enough space to compress the maximum sized sample */ { @@ -150,7 +150,7 @@ double compressWithDict(sampleInfo *srcInfo, dictInfo* dInfo, int compressionLev cctx = ZSTD_createCCtx(); if(!cctx || !dst) { cRatio = -1; - goto _nodictCleanup; + goto _cleanup; } /* Create CDict if there's a dictionary stored on buffer */ @@ -158,7 +158,7 @@ double compressWithDict(sampleInfo *srcInfo, dictInfo* dInfo, int compressionLev cdict = ZSTD_createCDict(dInfo->dictBuffer, dInfo->dictSize, compressionLevel); if(!cdict) { cRatio = -1; - goto _dictCleanup; + goto _cleanup; } } @@ -173,8 +173,7 @@ double compressWithDict(sampleInfo *srcInfo, dictInfo* dInfo, int compressionLev } if (ZSTD_isError(compressedSize)) { cRatio = -1; - if(hasDict) goto _dictCleanup; - else goto _nodictCleanup; + goto _cleanup; } totalCompressedSize += compressedSize; } @@ -189,14 +188,11 @@ double compressWithDict(sampleInfo *srcInfo, dictInfo* dInfo, int compressionLev DISPLAYLEVEL(2, "compressed size is %lu\n", totalCompressedSize); cRatio = (double)totalOriginalSize/(double)totalCompressedSize; -_dictCleanup: - ZSTD_freeCDict(cdict); - -_nodictCleanup: +_cleanup: free(dst); free(offsets); ZSTD_freeCCtx(cctx); - + ZSTD_freeCDict(cdict); return cRatio; } @@ -249,7 +245,7 @@ int benchmarkDictBuilder(sampleInfo *srcInfo, unsigned maxDictSize, ZDICT_random DISPLAYLEVEL(2, "%s took %f seconds to execute \n", name, timeSec); /* Calculate compression ratio */ - double cRatio = compressWithDict(srcInfo, dInfo, cLevel, displayLevel); + const double cRatio = compressWithDict(srcInfo, dInfo, cLevel, displayLevel); if (cRatio < 0) { DISPLAYLEVEL(1, "Compressing with %s dictionary does not work\n", name); result = 1; From 7f3f70f76621f4e488080d27f09614167c7b9a4b Mon Sep 17 00:00:00 2001 From: Jennifer Liu Date: Wed, 25 Jul 2018 16:34:07 -0700 Subject: [PATCH 072/372] Add Fast Cover Dictionary Builder --- .../fastCover/Makefile | 54 ++ .../fastCover/README.md | 24 + .../fastCover/fastCover.c | 738 ++++++++++++++++++ .../fastCover/fastCover.h | 47 ++ .../fastCover/main.c | 177 +++++ .../fastCover/test.sh | 14 + 6 files changed, 1054 insertions(+) create mode 100644 contrib/experimental_dict_builders/fastCover/Makefile create mode 100644 contrib/experimental_dict_builders/fastCover/README.md create mode 100644 contrib/experimental_dict_builders/fastCover/fastCover.c create mode 100644 contrib/experimental_dict_builders/fastCover/fastCover.h create mode 100644 contrib/experimental_dict_builders/fastCover/main.c create mode 100644 contrib/experimental_dict_builders/fastCover/test.sh diff --git a/contrib/experimental_dict_builders/fastCover/Makefile b/contrib/experimental_dict_builders/fastCover/Makefile new file mode 100644 index 000000000..9c56013d8 --- /dev/null +++ b/contrib/experimental_dict_builders/fastCover/Makefile @@ -0,0 +1,54 @@ +ARG := + +CC ?= gcc +CFLAGS ?= -O3 +INCLUDES := -I ../../../programs -I ../randomDictBuilder -I ../../../lib/common -I ../../../lib -I ../../../lib/dictBuilder + +IO_FILE := ../randomDictBuilder/io.c + +TEST_INPUT := ../../../lib +TEST_OUTPUT := fastCoverDict + +all: main run clean + +.PHONY: test +test: main testrun testshell clean + +.PHONY: run +run: + echo "Building a fastCover dictionary with given arguments" + ./main $(ARG) + +main: main.o io.o fastCover.o libzstd.a + $(CC) $(CFLAGS) main.o io.o fastCover.o libzstd.a -o main + +main.o: main.c + $(CC) $(CFLAGS) $(INCLUDES) -c main.c + +fastCover.o: fastCover.c + $(CC) $(CFLAGS) $(INCLUDES) -c fastCover.c + +io.o: $(IO_FILE) + $(CC) $(CFLAGS) $(INCLUDES) -c $(IO_FILE) + +libzstd.a: + $(MAKE) -C ../../../lib libzstd.a + mv ../../../lib/libzstd.a . + +.PHONY: testrun +testrun: main + echo "Run with $(TEST_INPUT) and $(TEST_OUTPUT) " + ./main in=$(TEST_INPUT) out=$(TEST_OUTPUT) + zstd -be3 -D $(TEST_OUTPUT) -r $(TEST_INPUT) -q + rm -f $(TEST_OUTPUT) + +.PHONY: testshell +testshell: test.sh + sh test.sh + echo "Finish running test.sh" + +.PHONY: clean +clean: + rm -f *.o main libzstd.a + $(MAKE) -C ../../../lib clean + echo "Cleaning is completed" diff --git a/contrib/experimental_dict_builders/fastCover/README.md b/contrib/experimental_dict_builders/fastCover/README.md new file mode 100644 index 000000000..088e38be7 --- /dev/null +++ b/contrib/experimental_dict_builders/fastCover/README.md @@ -0,0 +1,24 @@ +FastCover Dictionary Builder + +### Permitted Arguments: +Input File/Directory (in=fileName): required; file/directory used to build dictionary; if directory, will operate recursively for files inside directory; can include multiple files/directories, each following "in=" +Output Dictionary (out=dictName): if not provided, default to fastCoverDict +Dictionary ID (dictID=#): nonnegative number; if not provided, default to 0 +Maximum Dictionary Size (maxdict=#): positive number; in bytes, if not provided, default to 110KB +Size of Selected Segment (k=#): positive number; in bytes; if not provided, default to 200 +Size of Dmer (d=#): positive number; in bytes; if not provided, default to 8 +Number of steps (steps=#): positive number, if not provided, default to 32 +Percentage of samples used for training(split=#): positive number; if not provided, default to 100 + + +###Running Test: +make test + + +###Usage: +To build a random dictionary with the provided arguments: make ARG= followed by arguments + + +### Examples: +make ARG="in=../../../lib/dictBuilder out=dict100 dictID=520" +make ARG="in=../../../lib/dictBuilder in=../../../lib/compress" diff --git a/contrib/experimental_dict_builders/fastCover/fastCover.c b/contrib/experimental_dict_builders/fastCover/fastCover.c new file mode 100644 index 000000000..6d3ad90ab --- /dev/null +++ b/contrib/experimental_dict_builders/fastCover/fastCover.c @@ -0,0 +1,738 @@ +/*-************************************* +* Dependencies +***************************************/ +#include /* fprintf */ +#include /* malloc, free, qsort */ +#include /* memset */ +#include /* clock */ +#include "mem.h" /* read */ +#include "pool.h" +#include "threading.h" +#include "fastCover.h" +#include "zstd_internal.h" /* includes zstd.h */ +#include "zdict.h" + + +/*-************************************* +* Constants +***************************************/ +#define FASTCOVER_MAX_SAMPLES_SIZE (sizeof(size_t) == 8 ? ((U32)-1) : ((U32)1 GB)) +#define FASTCOVER_MAX_F 32 +#define DEFAULT_SPLITPOINT 1.0 + +/*-************************************* +* Console display +***************************************/ +static int g_displayLevel = 2; +#define DISPLAY(...) \ + { \ + fprintf(stderr, __VA_ARGS__); \ + fflush(stderr); \ + } +#define LOCALDISPLAYLEVEL(displayLevel, l, ...) \ + if (displayLevel >= l) { \ + DISPLAY(__VA_ARGS__); \ + } /* 0 : no display; 1: errors; 2: default; 3: details; 4: debug */ +#define DISPLAYLEVEL(l, ...) LOCALDISPLAYLEVEL(g_displayLevel, l, __VA_ARGS__) + +#define LOCALDISPLAYUPDATE(displayLevel, l, ...) \ + if (displayLevel >= l) { \ + if ((clock() - g_time > refreshRate) || (displayLevel >= 4)) { \ + g_time = clock(); \ + DISPLAY(__VA_ARGS__); \ + } \ + } +#define DISPLAYUPDATE(l, ...) LOCALDISPLAYUPDATE(g_displayLevel, l, __VA_ARGS__) +static const clock_t refreshRate = CLOCKS_PER_SEC * 15 / 100; +static clock_t g_time = 0; + + +/*-************************************* +* Hash Function +***************************************/ +static const U64 prime8bytes = 0xCF1BBCDCB7A56463ULL; +static size_t ZSTD_hash8(U64 u, U32 h) { return (size_t)(((u) * prime8bytes) >> (64-h)) ; } +static size_t ZSTD_hash8Ptr(const void* p, U32 h) { return ZSTD_hash8(MEM_readLE64(p), h); } + +/** + * Hash the 8-byte value pointed to by p and mod 2^f + */ +static size_t FASTCOVER_hash8PtrToIndex(const void* p, U32 h) { + return ZSTD_hash8Ptr(p, h) & ((1 << h) - 1); +} + + +/*-************************************* +* Context +***************************************/ +typedef struct { + const BYTE *samples; + size_t *offsets; + const size_t *samplesSizes; + size_t nbSamples; + size_t nbTrainSamples; + size_t nbTestSamples; + size_t nbDmers; + U32 *freqs; + unsigned d; +} FASTCOVER_ctx_t; + + +/*-************************************* +* Helper functions +***************************************/ +/** + * Returns the sum of the sample sizes. + */ +static size_t FASTCOVER_sum(const size_t *samplesSizes, unsigned nbSamples) { + size_t sum = 0; + unsigned i; + for (i = 0; i < nbSamples; ++i) { + sum += samplesSizes[i]; + } + return sum; +} + + +/*-************************************* +* fast functions +***************************************/ +/** + * A segment is a range in the source as well as the score of the segment. + */ +typedef struct { + U32 begin; + U32 end; + U32 score; +} FASTCOVER_segment_t; + + +/** + * Selects the best segment in an epoch. + * Segments of are scored according to the function: + * + * Let F(d) be the frequency of all dmers with hash value d. + * Let S_i be hash value of the dmer at position i of segment S which has length k. + * + * Score(S) = F(S_1) + F(S_2) + ... + F(S_{k-d+1}) + * + * Once the dmer with hash value d is in the dictionay we set F(d) = F(d)/2. + */ +static FASTCOVER_segment_t FASTCOVER_selectSegment(const FASTCOVER_ctx_t *ctx, + U32 *freqs, U32 begin,U32 end, + ZDICT_fastCover_params_t parameters) { + /* Constants */ + const U32 k = parameters.k; + const U32 d = parameters.d; + const U32 dmersInK = k - d + 1; + /* Try each segment (activeSegment) and save the best (bestSegment) */ + FASTCOVER_segment_t bestSegment = {0, 0, 0}; + FASTCOVER_segment_t activeSegment; + /* Reset the activeDmers in the segment */ + /* The activeSegment starts at the beginning of the epoch. */ + activeSegment.begin = begin; + activeSegment.end = begin; + activeSegment.score = 0; + /* Slide the activeSegment through the whole epoch. + * Save the best segment in bestSegment. + */ + while (activeSegment.end < end) { + /* Get hash value of current dmer */ + size_t index = FASTCOVER_hash8PtrToIndex(ctx->samples + activeSegment.end, parameters.f); + /* Add frequency of this index to score */ + activeSegment.score += freqs[index]; + /* Increment end of segment */ + activeSegment.end += 1; + /* If the window is now too large, drop the first position */ + if (activeSegment.end - activeSegment.begin == dmersInK + 1) { + /* Get hash value of the dmer to be eliminated from active segment */ + size_t delIndex = FASTCOVER_hash8PtrToIndex(ctx->samples + activeSegment.begin, parameters.f); + /* Subtract frequency of this index from score */ + activeSegment.score -= freqs[delIndex]; + /* Increment start of segment */ + activeSegment.begin += 1; + } + /* If this segment is the best so far save it */ + if (activeSegment.score > bestSegment.score) { + bestSegment = activeSegment; + } + } + { + /* Trim off the zero frequency head and tail from the segment. */ + U32 newBegin = bestSegment.end; + U32 newEnd = bestSegment.begin; + U32 pos; + for (pos = bestSegment.begin; pos != bestSegment.end; ++pos) { + size_t index = FASTCOVER_hash8PtrToIndex(ctx->samples + pos, parameters.f); + U32 freq = freqs[index]; + if (freq != 0) { + newBegin = MIN(newBegin, pos); + newEnd = pos + 1; + } + } + bestSegment.begin = newBegin; + bestSegment.end = newEnd; + } + { + /* Half the frequency of hash value of each dmer covered by the chosen segment. */ + U32 pos; + for (pos = bestSegment.begin; pos != bestSegment.end; ++pos) { + size_t i = FASTCOVER_hash8PtrToIndex(ctx->samples + pos, parameters.f); + freqs[i] = freqs[i]/2; + } + } + return bestSegment; +} + +/** + * Check the validity of the parameters. + * Returns non-zero if the parameters are valid and 0 otherwise. + */ +static int FASTCOVER_checkParameters(ZDICT_fastCover_params_t parameters, + size_t maxDictSize) { + /* k, d, and f are required parameters */ + if (parameters.d == 0 || parameters.k == 0 || parameters.f == 0) { + return 0; + } + /* 0 < f <= FASTCOVER_MAX_F */ + if (parameters.f > FASTCOVER_MAX_F) { + return 0; + } + /* k <= maxDictSize */ + if (parameters.k > maxDictSize) { + return 0; + } + /* d <= k */ + if (parameters.d > parameters.k) { + return 0; + } + /* 0 < splitPoint <= 1 */ + if (parameters.splitPoint <= 0 || parameters.splitPoint > 1) { + return 0; + } + return 1; +} + + +/** + * Clean up a context initialized with `FASTCOVER_ctx_init()`. + */ +static void FASTCOVER_ctx_destroy(FASTCOVER_ctx_t *ctx) { + if (!ctx) { + return; + } + if (ctx->freqs) { + free(ctx->freqs); + ctx->freqs = NULL; + } + if (ctx->offsets) { + free(ctx->offsets); + ctx->offsets = NULL; + } +} + +/** + * Calculate for frequency of hash value of each dmer in ctx->samples + */ +static void FASTCOVER_getFrequency(U32 *freqs, unsigned f, FASTCOVER_ctx_t *ctx){ + /* inCurrSample keeps track of this hash value has already be seen in previous dmers in the same sample*/ + size_t* inCurrSample = (size_t *)malloc((1<nbTrainSamples; i++) { + memset(inCurrSample, 0, (1 << f)); /* Reset inCurrSample for each sample */ + size_t currSampleStart = ctx->offsets[i]; + size_t currSampleEnd = ctx->offsets[i+1]; + start = currSampleStart; + while (start + f < currSampleEnd) { + size_t dmerIndex = FASTCOVER_hash8PtrToIndex(ctx->samples + start, f); + /* if no dmer with same hash value has been seen in current sample */ + if (inCurrSample[dmerIndex] == 0) { + inCurrSample[dmerIndex]++; + freqs[dmerIndex]++; + } + start++; + } + } + free(inCurrSample); +} + +/** + * Prepare a context for dictionary building. + * The context is only dependent on the parameter `d` and can used multiple + * times. + * Returns 1 on success or zero on error. + * The context must be destroyed with `FASTCOVER_ctx_destroy()`. + */ +static int FASTCOVER_ctx_init(FASTCOVER_ctx_t *ctx, const void *samplesBuffer, + const size_t *samplesSizes, unsigned nbSamples, + unsigned d, double splitPoint, unsigned f) { + const BYTE *const samples = (const BYTE *)samplesBuffer; + const size_t totalSamplesSize = FASTCOVER_sum(samplesSizes, nbSamples); + /* Split samples into testing and training sets */ + const unsigned nbTrainSamples = splitPoint < 1.0 ? (unsigned)((double)nbSamples * splitPoint) : nbSamples; + const unsigned nbTestSamples = splitPoint < 1.0 ? nbSamples - nbTrainSamples : nbSamples; + const size_t trainingSamplesSize = splitPoint < 1.0 ? FASTCOVER_sum(samplesSizes, nbTrainSamples) : totalSamplesSize; + const size_t testSamplesSize = splitPoint < 1.0 ? FASTCOVER_sum(samplesSizes + nbTrainSamples, nbTestSamples) : totalSamplesSize; + /* Checks */ + if (totalSamplesSize < MAX(d, sizeof(U64)) || + totalSamplesSize >= (size_t)FASTCOVER_MAX_SAMPLES_SIZE) { + DISPLAYLEVEL(1, "Total samples size is too large (%u MB), maximum size is %u MB\n", + (U32)(totalSamplesSize>>20), (FASTCOVER_MAX_SAMPLES_SIZE >> 20)); + return 0; + } + /* Check if there are at least 5 training samples */ + if (nbTrainSamples < 5) { + DISPLAYLEVEL(1, "Total number of training samples is %u and is invalid.", nbTrainSamples); + return 0; + } + /* Check if there's testing sample */ + if (nbTestSamples < 1) { + DISPLAYLEVEL(1, "Total number of testing samples is %u and is invalid.", nbTestSamples); + return 0; + } + /* Zero the context */ + memset(ctx, 0, sizeof(*ctx)); + DISPLAYLEVEL(2, "Training on %u samples of total size %u\n", nbTrainSamples, + (U32)trainingSamplesSize); + DISPLAYLEVEL(2, "Testing on %u samples of total size %u\n", nbTestSamples, + (U32)testSamplesSize); + + ctx->samples = samples; + ctx->samplesSizes = samplesSizes; + ctx->nbSamples = nbSamples; + ctx->nbTrainSamples = nbTrainSamples; + ctx->nbTestSamples = nbTestSamples; + ctx->nbDmers = trainingSamplesSize - d + 1; + ctx->d = d; + + /* The offsets of each file */ + ctx->offsets = (size_t *)malloc((nbSamples + 1) * sizeof(size_t)); + if (!ctx->offsets) { + DISPLAYLEVEL(1, "Failed to allocate scratch buffers\n"); + FASTCOVER_ctx_destroy(ctx); + return 0; + } + + /* Fill offsets from the samplesSizes */ + { + U32 i; + ctx->offsets[0] = 0; + for (i = 1; i <= nbSamples; ++i) { + ctx->offsets[i] = ctx->offsets[i - 1] + samplesSizes[i - 1]; + } + } + + /* Initialize frequency array of size 2^f */ + ctx->freqs =(U32 *)malloc((1 << f) * sizeof(U32)); + memset(ctx->freqs, 0, (1 << f) * sizeof(U32)); + + DISPLAYLEVEL(2, "Computing frequencies\n"); + FASTCOVER_getFrequency(ctx->freqs, f, ctx); + + return 1; +} + + +/** + * Given the prepared context build the dictionary. + */ +static size_t FASTCOVER_buildDictionary(const FASTCOVER_ctx_t *ctx, U32 *freqs, + void *dictBuffer, + size_t dictBufferCapacity, + ZDICT_fastCover_params_t parameters){ + BYTE *const dict = (BYTE *)dictBuffer; + size_t tail = dictBufferCapacity; + /* Divide the data up into epochs of equal size. + * We will select at least one segment from each epoch. + */ + const U32 epochs = MAX(1, (U32)(dictBufferCapacity / parameters.k)); + const U32 epochSize = (U32)(ctx->nbDmers / epochs); + size_t epoch; + DISPLAYLEVEL(2, "Breaking content into %u epochs of size %u\n", epochs, + epochSize); + /* Loop through the epochs until there are no more segments or the dictionary + * is full. + */ + for (epoch = 0; tail > 0; epoch = (epoch + 1) % epochs) { + const U32 epochBegin = (U32)(epoch * epochSize); + const U32 epochEnd = epochBegin + epochSize; + size_t segmentSize; + /* Select a segment */ + FASTCOVER_segment_t segment = FASTCOVER_selectSegment( + ctx, freqs, epochBegin, epochEnd, parameters); + + /* If the segment covers no dmers, then we are out of content */ + if (segment.score == 0) { + break; + } + + /* Trim the segment if necessary and if it is too small then we are done */ + segmentSize = MIN(segment.end - segment.begin + parameters.d - 1, tail); + if (segmentSize < parameters.d) { + break; + } + + /* We fill the dictionary from the back to allow the best segments to be + * referenced with the smallest offsets. + */ + tail -= segmentSize; + memcpy(dict + tail, ctx->samples + segment.begin, segmentSize); + DISPLAYUPDATE( + 2, "\r%u%% ", + (U32)(((dictBufferCapacity - tail) * 100) / dictBufferCapacity)); + } + DISPLAYLEVEL(2, "\r%79s\r", ""); + return tail; +} + + +/** + * FASTCOVER_best_t is used for two purposes: + * 1. Synchronizing threads. + * 2. Saving the best parameters and dictionary. + * + * All of the methods except FASTCOVER_best_init() are thread safe if zstd is + * compiled with multithreaded support. + */ +typedef struct fast_best_s { + ZSTD_pthread_mutex_t mutex; + ZSTD_pthread_cond_t cond; + size_t liveJobs; + void *dict; + size_t dictSize; + ZDICT_fastCover_params_t parameters; + size_t compressedSize; +} FASTCOVER_best_t; + +/** + * Initialize the `FASTCOVER_best_t`. + */ +static void FASTCOVER_best_init(FASTCOVER_best_t *best) { + if (best==NULL) return; /* compatible with init on NULL */ + (void)ZSTD_pthread_mutex_init(&best->mutex, NULL); + (void)ZSTD_pthread_cond_init(&best->cond, NULL); + best->liveJobs = 0; + best->dict = NULL; + best->dictSize = 0; + best->compressedSize = (size_t)-1; + memset(&best->parameters, 0, sizeof(best->parameters)); +} + +/** + * Wait until liveJobs == 0. + */ +static void FASTCOVER_best_wait(FASTCOVER_best_t *best) { + if (!best) { + return; + } + ZSTD_pthread_mutex_lock(&best->mutex); + while (best->liveJobs != 0) { + ZSTD_pthread_cond_wait(&best->cond, &best->mutex); + } + ZSTD_pthread_mutex_unlock(&best->mutex); +} + +/** + * Call FASTCOVER_best_wait() and then destroy the FASTCOVER_best_t. + */ +static void FASTCOVER_best_destroy(FASTCOVER_best_t *best) { + if (!best) { + return; + } + FASTCOVER_best_wait(best); + if (best->dict) { + free(best->dict); + } + ZSTD_pthread_mutex_destroy(&best->mutex); + ZSTD_pthread_cond_destroy(&best->cond); +} + +/** + * Called when a thread is about to be launched. + * Increments liveJobs. + */ +static void FASTCOVER_best_start(FASTCOVER_best_t *best) { + if (!best) { + return; + } + ZSTD_pthread_mutex_lock(&best->mutex); + ++best->liveJobs; + ZSTD_pthread_mutex_unlock(&best->mutex); +} + +/** + * Called when a thread finishes executing, both on error or success. + * Decrements liveJobs and signals any waiting threads if liveJobs == 0. + * If this dictionary is the best so far save it and its parameters. + */ +static void FASTCOVER_best_finish(FASTCOVER_best_t *best, size_t compressedSize, + ZDICT_fastCover_params_t parameters, void *dict, + size_t dictSize) { + if (!best) { + return; + } + { + size_t liveJobs; + ZSTD_pthread_mutex_lock(&best->mutex); + --best->liveJobs; + liveJobs = best->liveJobs; + /* If the new dictionary is better */ + if (compressedSize < best->compressedSize) { + /* Allocate space if necessary */ + if (!best->dict || best->dictSize < dictSize) { + if (best->dict) { + free(best->dict); + } + best->dict = malloc(dictSize); + if (!best->dict) { + best->compressedSize = ERROR(GENERIC); + best->dictSize = 0; + return; + } + } + /* Save the dictionary, parameters, and size */ + memcpy(best->dict, dict, dictSize); + best->dictSize = dictSize; + best->parameters = parameters; + best->compressedSize = compressedSize; + } + ZSTD_pthread_mutex_unlock(&best->mutex); + if (liveJobs == 0) { + ZSTD_pthread_cond_broadcast(&best->cond); + } + } +} + +/** + * Parameters for FASTCOVER_tryParameters(). + */ +typedef struct FASTCOVER_tryParameters_data_s { + const FASTCOVER_ctx_t *ctx; + FASTCOVER_best_t *best; + size_t dictBufferCapacity; + ZDICT_fastCover_params_t parameters; +} FASTCOVER_tryParameters_data_t; + +/** + * Tries a set of parameters and updates the FASTCOVER_best_t with the results. + * This function is thread safe if zstd is compiled with multithreaded support. + * It takes its parameters as an *OWNING* opaque pointer to support threading. + */ +static void FASTCOVER_tryParameters(void *opaque) { + /* Save parameters as local variables */ + FASTCOVER_tryParameters_data_t *const data = (FASTCOVER_tryParameters_data_t *)opaque; + const FASTCOVER_ctx_t *const ctx = data->ctx; + const ZDICT_fastCover_params_t parameters = data->parameters; + size_t dictBufferCapacity = data->dictBufferCapacity; + size_t totalCompressedSize = ERROR(GENERIC); + /* Allocate space for hash table, dict, and freqs */ + BYTE *const dict = (BYTE * const)malloc(dictBufferCapacity); + U32 *freqs = (U32*) malloc((1 << parameters.f) * sizeof(U32)); + if (!dict || !freqs) { + DISPLAYLEVEL(1, "Failed to allocate buffers: out of memory\n"); + goto _cleanup; + } + /* Copy the frequencies because we need to modify them */ + memcpy(freqs, ctx->freqs, (1 << parameters.f) * sizeof(U32)); + /* Build the dictionary */ + { + const size_t tail = FASTCOVER_buildDictionary(ctx, freqs, dict, + dictBufferCapacity, parameters); + + dictBufferCapacity = ZDICT_finalizeDictionary( + dict, dictBufferCapacity, dict + tail, dictBufferCapacity - tail, + ctx->samples, ctx->samplesSizes, (unsigned)ctx->nbTrainSamples, + parameters.zParams); + if (ZDICT_isError(dictBufferCapacity)) { + DISPLAYLEVEL(1, "Failed to finalize dictionary\n"); + goto _cleanup; + } + } + /* Check total compressed size */ + { + /* Pointers */ + ZSTD_CCtx *cctx; + ZSTD_CDict *cdict; + void *dst; + /* Local variables */ + size_t dstCapacity; + size_t i; + /* Allocate dst with enough space to compress the maximum sized sample */ + { + size_t maxSampleSize = 0; + i = parameters.splitPoint < 1.0 ? ctx->nbTrainSamples : 0; + for (; i < ctx->nbSamples; ++i) { + maxSampleSize = MAX(ctx->samplesSizes[i], maxSampleSize); + } + dstCapacity = ZSTD_compressBound(maxSampleSize); + dst = malloc(dstCapacity); + } + /* Create the cctx and cdict */ + cctx = ZSTD_createCCtx(); + cdict = ZSTD_createCDict(dict, dictBufferCapacity, + parameters.zParams.compressionLevel); + if (!dst || !cctx || !cdict) { + goto _compressCleanup; + } + /* Compress each sample and sum their sizes (or error) */ + totalCompressedSize = dictBufferCapacity; + i = parameters.splitPoint < 1.0 ? ctx->nbTrainSamples : 0; + for (; i < ctx->nbSamples; ++i) { + const size_t size = ZSTD_compress_usingCDict( + cctx, dst, dstCapacity, ctx->samples + ctx->offsets[i], + ctx->samplesSizes[i], cdict); + if (ZSTD_isError(size)) { + totalCompressedSize = ERROR(GENERIC); + goto _compressCleanup; + } + totalCompressedSize += size; + } + _compressCleanup: + ZSTD_freeCCtx(cctx); + ZSTD_freeCDict(cdict); + if (dst) { + free(dst); + } + } + +_cleanup: + FASTCOVER_best_finish(data->best, totalCompressedSize, parameters, dict, + dictBufferCapacity); + free(data); + if (dict) { + free(dict); + } + if (freqs) { + free(freqs); + } +} + +ZDICTLIB_API size_t ZDICT_optimizeTrainFromBuffer_fastCover( + void *dictBuffer, size_t dictBufferCapacity, const void *samplesBuffer, + const size_t *samplesSizes, unsigned nbSamples, + ZDICT_fastCover_params_t *parameters) { + /* constants */ + const unsigned nbThreads = parameters->nbThreads; + const double splitPoint = + parameters->splitPoint <= 0.0 ? DEFAULT_SPLITPOINT : parameters->splitPoint; + const unsigned kMinD = parameters->d == 0 ? 8 : parameters->d; + const unsigned kMaxD = parameters->d == 0 ? 8 : parameters->d; + const unsigned kMinK = parameters->k == 0 ? 50 : parameters->k; + const unsigned kMaxK = parameters->k == 0 ? 2000 : parameters->k; + const unsigned kSteps = parameters->steps == 0 ? 40 : parameters->steps; + const unsigned kStepSize = MAX((kMaxK - kMinK) / kSteps, 1); + const unsigned kIterations = + (1 + (kMaxD - kMinD) / 2) * (1 + (kMaxK - kMinK) / kStepSize); + const unsigned f = parameters->f == 0 ? 23 : parameters->f; + + /* Local variables */ + const int displayLevel = parameters->zParams.notificationLevel; + unsigned iteration = 1; + unsigned d; + unsigned k; + FASTCOVER_best_t best; + POOL_ctx *pool = NULL; + + /* Checks */ + if (splitPoint <= 0 || splitPoint > 1) { + LOCALDISPLAYLEVEL(displayLevel, 1, "Incorrect parameters\n"); + return ERROR(GENERIC); + } + if (kMinK < kMaxD || kMaxK < kMinK) { + LOCALDISPLAYLEVEL(displayLevel, 1, "Incorrect parameters\n"); + return ERROR(GENERIC); + } + if (nbSamples == 0) { + DISPLAYLEVEL(1, "fast must have at least one input file\n"); + return ERROR(GENERIC); + } + if (dictBufferCapacity < ZDICT_DICTSIZE_MIN) { + DISPLAYLEVEL(1, "dictBufferCapacity must be at least %u\n", + ZDICT_DICTSIZE_MIN); + return ERROR(dstSize_tooSmall); + } + if (nbThreads > 1) { + pool = POOL_create(nbThreads, 1); + if (!pool) { + return ERROR(memory_allocation); + } + } + /* Initialization */ + FASTCOVER_best_init(&best); + /* Turn down global display level to clean up display at level 2 and below */ + g_displayLevel = displayLevel == 0 ? 0 : displayLevel - 1; + /* Loop through d first because each new value needs a new context */ + LOCALDISPLAYLEVEL(displayLevel, 2, "Trying %u different sets of parameters\n", + kIterations); + for (d = kMinD; d <= kMaxD; d += 2) { + /* Initialize the context for this value of d */ + FASTCOVER_ctx_t ctx; + LOCALDISPLAYLEVEL(displayLevel, 3, "d=%u\n", d); + if (!FASTCOVER_ctx_init(&ctx, samplesBuffer, samplesSizes, nbSamples, d, splitPoint, f)) { + LOCALDISPLAYLEVEL(displayLevel, 1, "Failed to initialize context\n"); + FASTCOVER_best_destroy(&best); + POOL_free(pool); + return ERROR(GENERIC); + } + /* Loop through k reusing the same context */ + for (k = kMinK; k <= kMaxK; k += kStepSize) { + /* Prepare the arguments */ + FASTCOVER_tryParameters_data_t *data = (FASTCOVER_tryParameters_data_t *)malloc( + sizeof(FASTCOVER_tryParameters_data_t)); + LOCALDISPLAYLEVEL(displayLevel, 3, "k=%u\n", k); + if (!data) { + LOCALDISPLAYLEVEL(displayLevel, 1, "Failed to allocate parameters\n"); + FASTCOVER_best_destroy(&best); + FASTCOVER_ctx_destroy(&ctx); + POOL_free(pool); + return ERROR(GENERIC); + } + data->ctx = &ctx; + data->best = &best; + data->dictBufferCapacity = dictBufferCapacity; + data->parameters = *parameters; + data->parameters.k = k; + data->parameters.d = d; + data->parameters.f = f; + data->parameters.splitPoint = splitPoint; + data->parameters.steps = kSteps; + data->parameters.zParams.notificationLevel = g_displayLevel; + /* Check the parameters */ + if (!FASTCOVER_checkParameters(data->parameters, dictBufferCapacity)) { + DISPLAYLEVEL(1, "fastCover parameters incorrect\n"); + free(data); + continue; + } + /* Call the function and pass ownership of data to it */ + FASTCOVER_best_start(&best); + if (pool) { + POOL_add(pool, &FASTCOVER_tryParameters, data); + } else { + FASTCOVER_tryParameters(data); + } + /* Print status */ + LOCALDISPLAYUPDATE(displayLevel, 2, "\r%u%% ", + (U32)((iteration * 100) / kIterations)); + ++iteration; + } + FASTCOVER_best_wait(&best); + FASTCOVER_ctx_destroy(&ctx); + } + LOCALDISPLAYLEVEL(displayLevel, 2, "\r%79s\r", ""); + /* Fill the output buffer and parameters with output of the best parameters */ + { + const size_t dictSize = best.dictSize; + if (ZSTD_isError(best.compressedSize)) { + const size_t compressedSize = best.compressedSize; + FASTCOVER_best_destroy(&best); + POOL_free(pool); + return compressedSize; + } + *parameters = best.parameters; + memcpy(dictBuffer, best.dict, dictSize); + FASTCOVER_best_destroy(&best); + POOL_free(pool); + return dictSize; + } + +} diff --git a/contrib/experimental_dict_builders/fastCover/fastCover.h b/contrib/experimental_dict_builders/fastCover/fastCover.h new file mode 100644 index 000000000..eca04baab --- /dev/null +++ b/contrib/experimental_dict_builders/fastCover/fastCover.h @@ -0,0 +1,47 @@ +#include /* fprintf */ +#include /* malloc, free, qsort */ +#include /* memset */ +#include /* clock */ +#include "mem.h" /* read */ +#include "pool.h" +#include "threading.h" +#include "zstd_internal.h" /* includes zstd.h */ +#ifndef ZDICT_STATIC_LINKING_ONLY +#define ZDICT_STATIC_LINKING_ONLY +#endif +#include "zdict.h" + + + + + +typedef struct { + unsigned k; /* Segment size : constraint: 0 < k : Reasonable range [16, 2048+] */ + unsigned d; /* dmer size : constraint: 0 < d <= k : Reasonable range [6, 16] */ + unsigned f; /* log of size of frequency array */ + unsigned steps; /* Number of steps : Only used for optimization : 0 means default (32) : Higher means more parameters checked */ + unsigned nbThreads; /* Number of threads : constraint: 0 < nbThreads : 1 means single-threaded : Only used for optimization : Ignored if ZSTD_MULTITHREAD is not defined */ + double splitPoint; /* Percentage of samples used for training: the first nbSamples * splitPoint samples will be used to training, the last nbSamples * (1 - splitPoint) samples will be used for testing, 0 means default (1.0), 1.0 when all samples are used for both training and testing */ + ZDICT_params_t zParams; +} ZDICT_fastCover_params_t; + + + +/*! ZDICT_optimizeTrainFromBuffer_fastCover(): + * Train a dictionary from an array of samples using a modified version of the COVER algorithm. + * Samples must be stored concatenated in a single flat buffer `samplesBuffer`, + * supplied with an array of sizes `samplesSizes`, providing the size of each sample, in order. + * The resulting dictionary will be saved into `dictBuffer`. + * All of the parameters except for f are optional. + * If d is non-zero then we don't check multiple values of d, otherwise we check d = {6, 8, 10, 12, 14, 16}. + * if steps is zero it defaults to its default value. + * If k is non-zero then we don't check multiple values of k, otherwise we check steps values in [16, 2048]. + * + * @return: size of dictionary stored into `dictBuffer` (<= `dictBufferCapacity`) + * or an error code, which can be tested with ZDICT_isError(). + * On success `*parameters` contains the parameters selected. + */ +ZDICTLIB_API size_t ZDICT_optimizeTrainFromBuffer_fastCover( + void *dictBuffer, size_t dictBufferCapacity, const void *samplesBuffer, + const size_t *samplesSizes, unsigned nbSamples, + ZDICT_fastCover_params_t *parameters); diff --git a/contrib/experimental_dict_builders/fastCover/main.c b/contrib/experimental_dict_builders/fastCover/main.c new file mode 100644 index 000000000..260eeb281 --- /dev/null +++ b/contrib/experimental_dict_builders/fastCover/main.c @@ -0,0 +1,177 @@ +#include /* fprintf */ +#include /* malloc, free, qsort */ +#include /* strcmp, strlen */ +#include /* errno */ +#include +#include "fastCover.h" +#include "io.h" +#include "util.h" +#include "zdict.h" + + +/*-************************************* +* Console display +***************************************/ +#define DISPLAY(...) fprintf(stderr, __VA_ARGS__) +#define DISPLAYLEVEL(l, ...) if (displayLevel>=l) { DISPLAY(__VA_ARGS__); } + +static const U64 g_refreshRate = SEC_TO_MICRO / 6; +static UTIL_time_t g_displayClock = UTIL_TIME_INITIALIZER; + +#define DISPLAYUPDATE(l, ...) { if (displayLevel>=l) { \ + if ((UTIL_clockSpanMicro(g_displayClock) > g_refreshRate) || (displayLevel>=4)) \ + { g_displayClock = UTIL_getTime(); DISPLAY(__VA_ARGS__); \ + if (displayLevel>=4) fflush(stderr); } } } + + +/*-************************************* +* Exceptions +***************************************/ +#ifndef DEBUG +# define DEBUG 0 +#endif +#define DEBUGOUTPUT(...) if (DEBUG) DISPLAY(__VA_ARGS__); +#define EXM_THROW(error, ...) \ +{ \ + DEBUGOUTPUT("Error defined at %s, line %i : \n", __FILE__, __LINE__); \ + DISPLAY("Error %i : ", error); \ + DISPLAY(__VA_ARGS__); \ + DISPLAY("\n"); \ + exit(error); \ +} + + +/*-************************************* +* Constants +***************************************/ +static const unsigned g_defaultMaxDictSize = 110 KB; +#define DEFAULT_CLEVEL 3 + + +/*-************************************* +* FASTCOVER +***************************************/ +int FASTCOVER_trainFromFiles(const char* dictFileName, sampleInfo *info, + unsigned maxDictSize, + ZDICT_fastCover_params_t *params) { + unsigned const displayLevel = params->zParams.notificationLevel; + void* const dictBuffer = malloc(maxDictSize); + + int result = 0; + + /* Checks */ + if (!dictBuffer) + EXM_THROW(12, "not enough memory for trainFromFiles"); /* should not happen */ + + { size_t dictSize; + dictSize = ZDICT_optimizeTrainFromBuffer_fastCover(dictBuffer, maxDictSize, info->srcBuffer, + info->samplesSizes, info->nbSamples, params); + DISPLAYLEVEL(2, "k=%u\nd=%u\nf=%u\nsteps=%u\nsplit=%u\n", params->k, params->d, params->f, params->steps, (unsigned)(params->splitPoint*100)); + if (ZDICT_isError(dictSize)) { + DISPLAYLEVEL(1, "dictionary training failed : %s \n", ZDICT_getErrorName(dictSize)); /* should not happen */ + result = 1; + goto _done; + } + /* save dict */ + DISPLAYLEVEL(2, "Save dictionary of size %u into file %s \n", (U32)dictSize, dictFileName); + saveDict(dictFileName, dictBuffer, dictSize); + } + + /* clean up */ +_done: + free(dictBuffer); + return result; +} + + + +int main(int argCount, const char* argv[]) +{ + int displayLevel = 2; + const char* programName = argv[0]; + int operationResult = 0; + + /* Initialize arguments to default values */ + unsigned k = 200; + unsigned d = 8; + unsigned f = 23; + unsigned steps = 32; + unsigned nbThreads = 1; + unsigned split = 100; + const char* outputFile = "fastCoverDict"; + unsigned dictID = 0; + unsigned maxDictSize = g_defaultMaxDictSize; + + /* Initialize table to store input files */ + const char** filenameTable = (const char**)malloc(argCount * sizeof(const char*)); + unsigned filenameIdx = 0; + + char* fileNamesBuf = NULL; + unsigned fileNamesNb = filenameIdx; + int followLinks = 0; /* follow directory recursively */ + const char** extendedFileList = NULL; + + /* Parse arguments */ + for (int i = 1; i < argCount; i++) { + const char* argument = argv[i]; + if (longCommandWArg(&argument, "k=")) { k = readU32FromChar(&argument); continue; } + if (longCommandWArg(&argument, "d=")) { d = readU32FromChar(&argument); continue; } + if (longCommandWArg(&argument, "f=")) { f = readU32FromChar(&argument); continue; } + if (longCommandWArg(&argument, "steps=")) { steps = readU32FromChar(&argument); continue; } + if (longCommandWArg(&argument, "split=")) { split = readU32FromChar(&argument); continue; } + if (longCommandWArg(&argument, "dictID=")) { dictID = readU32FromChar(&argument); continue; } + if (longCommandWArg(&argument, "maxdict=")) { maxDictSize = readU32FromChar(&argument); continue; } + if (longCommandWArg(&argument, "in=")) { + filenameTable[filenameIdx] = argument; + filenameIdx++; + continue; + } + if (longCommandWArg(&argument, "out=")) { + outputFile = argument; + continue; + } + DISPLAYLEVEL(1, "Incorrect parameters\n"); + operationResult = 1; + return operationResult; + } + + /* Get the list of all files recursively (because followLinks==0)*/ + extendedFileList = UTIL_createFileList(filenameTable, filenameIdx, &fileNamesBuf, + &fileNamesNb, followLinks); + if (extendedFileList) { + unsigned u; + for (u=0; u Date: Wed, 25 Jul 2018 16:54:08 -0700 Subject: [PATCH 073/372] Make hash value const --- .../experimental_dict_builders/fastCover/fastCover.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/contrib/experimental_dict_builders/fastCover/fastCover.c b/contrib/experimental_dict_builders/fastCover/fastCover.c index 6d3ad90ab..32a15a4bf 100644 --- a/contrib/experimental_dict_builders/fastCover/fastCover.c +++ b/contrib/experimental_dict_builders/fastCover/fastCover.c @@ -138,7 +138,7 @@ static FASTCOVER_segment_t FASTCOVER_selectSegment(const FASTCOVER_ctx_t *ctx, */ while (activeSegment.end < end) { /* Get hash value of current dmer */ - size_t index = FASTCOVER_hash8PtrToIndex(ctx->samples + activeSegment.end, parameters.f); + const size_t index = FASTCOVER_hash8PtrToIndex(ctx->samples + activeSegment.end, parameters.f); /* Add frequency of this index to score */ activeSegment.score += freqs[index]; /* Increment end of segment */ @@ -146,7 +146,7 @@ static FASTCOVER_segment_t FASTCOVER_selectSegment(const FASTCOVER_ctx_t *ctx, /* If the window is now too large, drop the first position */ if (activeSegment.end - activeSegment.begin == dmersInK + 1) { /* Get hash value of the dmer to be eliminated from active segment */ - size_t delIndex = FASTCOVER_hash8PtrToIndex(ctx->samples + activeSegment.begin, parameters.f); + const size_t delIndex = FASTCOVER_hash8PtrToIndex(ctx->samples + activeSegment.begin, parameters.f); /* Subtract frequency of this index from score */ activeSegment.score -= freqs[delIndex]; /* Increment start of segment */ @@ -163,7 +163,7 @@ static FASTCOVER_segment_t FASTCOVER_selectSegment(const FASTCOVER_ctx_t *ctx, U32 newEnd = bestSegment.begin; U32 pos; for (pos = bestSegment.begin; pos != bestSegment.end; ++pos) { - size_t index = FASTCOVER_hash8PtrToIndex(ctx->samples + pos, parameters.f); + const size_t index = FASTCOVER_hash8PtrToIndex(ctx->samples + pos, parameters.f); U32 freq = freqs[index]; if (freq != 0) { newBegin = MIN(newBegin, pos); @@ -177,7 +177,7 @@ static FASTCOVER_segment_t FASTCOVER_selectSegment(const FASTCOVER_ctx_t *ctx, /* Half the frequency of hash value of each dmer covered by the chosen segment. */ U32 pos; for (pos = bestSegment.begin; pos != bestSegment.end; ++pos) { - size_t i = FASTCOVER_hash8PtrToIndex(ctx->samples + pos, parameters.f); + const size_t i = FASTCOVER_hash8PtrToIndex(ctx->samples + pos, parameters.f); freqs[i] = freqs[i]/2; } } @@ -244,7 +244,7 @@ static void FASTCOVER_getFrequency(U32 *freqs, unsigned f, FASTCOVER_ctx_t *ctx) size_t currSampleEnd = ctx->offsets[i+1]; start = currSampleStart; while (start + f < currSampleEnd) { - size_t dmerIndex = FASTCOVER_hash8PtrToIndex(ctx->samples + start, f); + const size_t dmerIndex = FASTCOVER_hash8PtrToIndex(ctx->samples + start, f); /* if no dmer with same hash value has been seen in current sample */ if (inCurrSample[dmerIndex] == 0) { inCurrSample[dmerIndex]++; From d1fc507ef998f511f6f1da7edc57670bb6b3404f Mon Sep 17 00:00:00 2001 From: Jennifer Liu Date: Wed, 25 Jul 2018 17:05:54 -0700 Subject: [PATCH 074/372] Initial benchmarking result for fastCover --- .../benchmarkDictBuilder/Makefile | 10 +++-- .../benchmarkDictBuilder/README.md | 40 ++++++++++-------- .../benchmarkDictBuilder/benchmark.c | 42 +++++++++++++++---- 3 files changed, 62 insertions(+), 30 deletions(-) diff --git a/contrib/experimental_dict_builders/benchmarkDictBuilder/Makefile b/contrib/experimental_dict_builders/benchmarkDictBuilder/Makefile index 72ce04f2a..681494888 100644 --- a/contrib/experimental_dict_builders/benchmarkDictBuilder/Makefile +++ b/contrib/experimental_dict_builders/benchmarkDictBuilder/Makefile @@ -2,9 +2,10 @@ ARG := CC ?= gcc CFLAGS ?= -O3 -INCLUDES := -I ../randomDictBuilder -I ../../../programs -I ../../../lib/common -I ../../../lib -I ../../../lib/dictBuilder +INCLUDES := -I ../randomDictBuilder -I ../fastCover -I ../../../programs -I ../../../lib/common -I ../../../lib -I ../../../lib/dictBuilder RANDOM_FILE := ../randomDictBuilder/random.c +FAST_FILE := ../fastCover/fastCover.c IO_FILE := ../randomDictBuilder/io.c all: run clean @@ -21,8 +22,8 @@ test: benchmarkTest clean benchmarkTest: benchmark test.sh sh test.sh -benchmark: benchmark.o io.o random.o libzstd.a - $(CC) $(CFLAGS) benchmark.o io.o random.o libzstd.a -o benchmark +benchmark: benchmark.o io.o random.o fastCover.o libzstd.a + $(CC) $(CFLAGS) benchmark.o io.o random.o fastCover.o libzstd.a -o benchmark benchmark.o: benchmark.c $(CC) $(CFLAGS) $(INCLUDES) -c benchmark.c @@ -30,6 +31,9 @@ benchmark.o: benchmark.c random.o: $(RANDOM_FILE) $(CC) $(CFLAGS) $(INCLUDES) -c $(RANDOM_FILE) +fastCover.o: $(FAST_FILE) + $(CC) $(CFLAGS) $(INCLUDES) -c $(FAST_FILE) + io.o: $(IO_FILE) $(CC) $(CFLAGS) $(INCLUDES) -c $(IO_FILE) diff --git a/contrib/experimental_dict_builders/benchmarkDictBuilder/README.md b/contrib/experimental_dict_builders/benchmarkDictBuilder/README.md index de783a0ec..e02d592c4 100644 --- a/contrib/experimental_dict_builders/benchmarkDictBuilder/README.md +++ b/contrib/experimental_dict_builders/benchmarkDictBuilder/README.md @@ -18,30 +18,34 @@ github: | Algorithm | Speed(sec) | Compression Ratio | | ------------- |:-------------:| ------------------:| | nodict | 0.000004 | 2.999642 | -| random | 0.180238 | 8.786957 | -| cover | 33.891987 | 10.430999 | -| legacy | 1.077569 | 8.989482 | +| random | 0.135459 | 8.786957 | +| cover | 50.341079 | 10.641263 | +| legacy | 0.866283 | 8.989482 | +| fastCover | 13.450947 | 10.215174 | hg-commands | Algorithm | Speed(sec) | Compression Ratio | | ------------- |:-------------:| ------------------:| -| nodict | 0.000006 | 2.425291 | -| random | 0.088735 | 3.489515 | -| cover | 35.447300 | 4.030274 | -| legacy | 1.048509 | 3.911896 | +| nodict | 0.000020 | 2.425291 | +| random | 0.088828 | 3.489515 | +| cover | 60.028672 | 4.131136 | +| legacy | 0.852481 | 3.911896 | +| fastCover | 9.524284 | 3.977229 | + +hg-changelog +| Algorithm | Speed(sec) | Compression Ratio | +| ------------- |:-------------:| ------------------:| +| nodict | 0.000004 | 1.377613 | +| random | 0.621812 | 2.096785 | +| cover | 217.510962 | 2.188654 | +| legacy | 2.559194 | 2.058273 | +| fastCover | 51.132516 | 2.124185 | hg-manifest | Algorithm | Speed(sec) | Compression Ratio | | ------------- |:-------------:| ------------------:| | nodict | 0.000005 | 1.866385 | -| random | 1.148231 | 2.309485 | -| cover | 509.685257 | 2.575331 | -| legacy | 10.705866 | 2.506775 | - -hg-changelog -| Algorithm | Speed(sec) | Compression Ratio | -| ------------- |:-------------:| ------------------:| -| nodict | 0.000005 | 1.377613 | -| random | 0.706434 | 2.096785 | -| cover | 122.815783 | 2.175706 | -| legacy | 3.010318 | 2.058273 | +| random | 1.035220 | 2.309485 | +| cover | 930.480173 | 2.582597 | +| legacy | 8.916513 | 2.506775 | +| fastCover | 116.871089 | 2.525689 | diff --git a/contrib/experimental_dict_builders/benchmarkDictBuilder/benchmark.c b/contrib/experimental_dict_builders/benchmarkDictBuilder/benchmark.c index 640419649..865ecb34d 100644 --- a/contrib/experimental_dict_builders/benchmarkDictBuilder/benchmark.c +++ b/contrib/experimental_dict_builders/benchmarkDictBuilder/benchmark.c @@ -5,6 +5,7 @@ #include #include #include "random.h" +#include "fastCover.h" #include "dictBuilder.h" #include "zstd_internal.h" /* includes zstd.h */ #include "io.h" @@ -71,10 +72,11 @@ typedef struct { */ dictInfo* createDictFromFiles(sampleInfo *info, unsigned maxDictSize, ZDICT_random_params_t *randomParams, ZDICT_cover_params_t *coverParams, - ZDICT_legacy_params_t *legacyParams) { + ZDICT_legacy_params_t *legacyParams, ZDICT_fastCover_params_t *fastParams) { unsigned const displayLevel = randomParams ? randomParams->zParams.notificationLevel : coverParams ? coverParams->zParams.notificationLevel : legacyParams ? legacyParams->zParams.notificationLevel : + fastParams ? fastParams->zParams.notificationLevel : DEFAULT_DISPLAYLEVEL; /* no dict */ void* const dictBuffer = malloc(maxDictSize); @@ -94,6 +96,9 @@ dictInfo* createDictFromFiles(sampleInfo *info, unsigned maxDictSize, } else if(legacyParams) { dictSize = ZDICT_trainFromBuffer_legacy(dictBuffer, maxDictSize, info->srcBuffer, info->samplesSizes, info->nbSamples, *legacyParams); + } else if(fastParams) { + dictSize = ZDICT_optimizeTrainFromBuffer_fastCover(dictBuffer, maxDictSize, info->srcBuffer, + info->samplesSizes, info->nbSamples, fastParams); } else { dictSize = 0; } @@ -216,25 +221,29 @@ void freeDictInfo(dictInfo* info) { * @return 0 if benchmark successfully, 1 otherwise */ int benchmarkDictBuilder(sampleInfo *srcInfo, unsigned maxDictSize, ZDICT_random_params_t *randomParam, - ZDICT_cover_params_t *coverParam, ZDICT_legacy_params_t *legacyParam) { + ZDICT_cover_params_t *coverParam, ZDICT_legacy_params_t *legacyParam, + ZDICT_fastCover_params_t *fastParam) { /* Local variables */ const unsigned displayLevel = randomParam ? randomParam->zParams.notificationLevel : coverParam ? coverParam->zParams.notificationLevel : legacyParam ? legacyParam->zParams.notificationLevel : + fastParam ? fastParam->zParams.notificationLevel: DEFAULT_DISPLAYLEVEL; /* no dict */ const char* name = randomParam ? "RANDOM" : coverParam ? "COVER" : legacyParam ? "LEGACY" : + fastParam ? "FAST": "NODICT"; /* no dict */ const unsigned cLevel = randomParam ? randomParam->zParams.compressionLevel : coverParam ? coverParam->zParams.compressionLevel : legacyParam ? legacyParam->zParams.compressionLevel : + fastParam ? fastParam->zParams.compressionLevel: DEFAULT_CLEVEL; /* no dict */ int result = 0; /* Calculate speed */ const UTIL_time_t begin = UTIL_getTime(); - dictInfo* dInfo = createDictFromFiles(srcInfo, maxDictSize, randomParam, coverParam, legacyParam); + dictInfo* dInfo = createDictFromFiles(srcInfo, maxDictSize, randomParam, coverParam, legacyParam, fastParam); const U64 timeMicro = UTIL_clockSpanMicro(begin); const double timeSec = timeMicro / (double)SEC_TO_MICRO; if (!dInfo) { @@ -269,7 +278,6 @@ int main(int argCount, const char* argv[]) /* Initialize arguments to default values */ const unsigned k = 200; - const unsigned d = 6; const unsigned cLevel = DEFAULT_CLEVEL; const unsigned dictID = 0; const unsigned maxDictSize = g_defaultMaxDictSize; @@ -319,7 +327,7 @@ int main(int argCount, const char* argv[]) /* with no dict */ { - const int noDictResult = benchmarkDictBuilder(srcInfo, maxDictSize, NULL, NULL, NULL); + const int noDictResult = benchmarkDictBuilder(srcInfo, maxDictSize, NULL, NULL, NULL, NULL); if(noDictResult) { result = 1; goto _cleanup; @@ -331,7 +339,7 @@ int main(int argCount, const char* argv[]) ZDICT_random_params_t randomParam; randomParam.zParams = zParams; randomParam.k = k; - const int randomResult = benchmarkDictBuilder(srcInfo, maxDictSize, &randomParam, NULL, NULL); + const int randomResult = benchmarkDictBuilder(srcInfo, maxDictSize, &randomParam, NULL, NULL, NULL); if(randomResult) { result = 1; goto _cleanup; @@ -344,10 +352,9 @@ int main(int argCount, const char* argv[]) memset(&coverParam, 0, sizeof(coverParam)); coverParam.zParams = zParams; coverParam.splitPoint = 1.0; - coverParam.d = d; coverParam.steps = 40; coverParam.nbThreads = 1; - const int coverOptResult = benchmarkDictBuilder(srcInfo, maxDictSize, NULL, &coverParam, NULL); + const int coverOptResult = benchmarkDictBuilder(srcInfo, maxDictSize, NULL, &coverParam, NULL, NULL); if(coverOptResult) { result = 1; goto _cleanup; @@ -359,13 +366,30 @@ int main(int argCount, const char* argv[]) ZDICT_legacy_params_t legacyParam; legacyParam.zParams = zParams; legacyParam.selectivityLevel = 9; - const int legacyResult = benchmarkDictBuilder(srcInfo, maxDictSize, NULL, NULL, &legacyParam); + const int legacyResult = benchmarkDictBuilder(srcInfo, maxDictSize, NULL, NULL, &legacyParam, NULL); if(legacyResult) { result = 1; goto _cleanup; } } + /* for fastCover */ + { + ZDICT_fastCover_params_t fastParam; + memset(&fastParam, 0, sizeof(fastParam)); + fastParam.zParams = zParams; + fastParam.splitPoint = 1.0; + fastParam.d = 8; + fastParam.f = 23; + fastParam.steps = 40; + fastParam.nbThreads = 1; + const int fastOptResult = benchmarkDictBuilder(srcInfo, maxDictSize, NULL, NULL, NULL, &fastParam); + if(fastOptResult) { + result = 1; + goto _cleanup; + } + } + /* Free allocated memory */ _cleanup: UTIL_freeFileList(extendedFileList, fileNamesBuf); From 1e85f314d859c5295f88c98fcd0dc9fa03f68b12 Mon Sep 17 00:00:00 2001 From: Jennifer Liu Date: Wed, 25 Jul 2018 17:53:38 -0700 Subject: [PATCH 075/372] Benchmark fast cover optimize vs k=200 --- .../benchmarkDictBuilder/README.md | 60 ++++++++++--------- .../benchmarkDictBuilder/benchmark.c | 22 ++++++- 2 files changed, 53 insertions(+), 29 deletions(-) diff --git a/contrib/experimental_dict_builders/benchmarkDictBuilder/README.md b/contrib/experimental_dict_builders/benchmarkDictBuilder/README.md index e02d592c4..478d8793e 100644 --- a/contrib/experimental_dict_builders/benchmarkDictBuilder/README.md +++ b/contrib/experimental_dict_builders/benchmarkDictBuilder/README.md @@ -15,37 +15,41 @@ make ARG="in=../../../lib/dictBuilder in=../../../lib/compress" ###Benchmarking Result: github: -| Algorithm | Speed(sec) | Compression Ratio | -| ------------- |:-------------:| ------------------:| -| nodict | 0.000004 | 2.999642 | -| random | 0.135459 | 8.786957 | -| cover | 50.341079 | 10.641263 | -| legacy | 0.866283 | 8.989482 | -| fastCover | 13.450947 | 10.215174 | +| Algorithm | Speed(sec) | Compression Ratio | +| ------------------|:-------------:| ------------------:| +| nodict | 0.000004 | 2.999642 | +| random | 0.148247 | 8.786957 | +| cover | 56.331553 | 10.641263 | +| legacy | 0.917595 | 8.989482 | +| fastCover(opt) | 13.169979 | 10.215174 | +| fastCover(k=200) | 2.692406 | 8.657219 | hg-commands -| Algorithm | Speed(sec) | Compression Ratio | -| ------------- |:-------------:| ------------------:| -| nodict | 0.000020 | 2.425291 | -| random | 0.088828 | 3.489515 | -| cover | 60.028672 | 4.131136 | -| legacy | 0.852481 | 3.911896 | -| fastCover | 9.524284 | 3.977229 | +| Algorithm | Speed(sec) | Compression Ratio | +| ----------------- |:-------------:| ------------------:| +| nodict | 0.000007 | 2.425291 | +| random | 0.093990 | 3.489515 | +| cover | 58.602385 | 4.131136 | +| legacy | 0.865683 | 3.911896 | +| fastCover(opt) | 9.404134 | 3.977229 | +| fastCover(k=200) | 1.037434 | 3.810326 | hg-changelog -| Algorithm | Speed(sec) | Compression Ratio | -| ------------- |:-------------:| ------------------:| -| nodict | 0.000004 | 1.377613 | -| random | 0.621812 | 2.096785 | -| cover | 217.510962 | 2.188654 | -| legacy | 2.559194 | 2.058273 | -| fastCover | 51.132516 | 2.124185 | +| Algorithm | Speed(sec) | Compression Ratio | +| ----------------- |:-------------:| ------------------:| +| nodict | 0.000022 | 1.377613 | +| random | 0.551539 | 2.096785 | +| cover | 221.370056 | 2.188654 | +| legacy | 2.405923 | 2.058273 | +| fastCover(opt) | 49.526246 | 2.124185 | +| fastCover(k=200) | 9.746872 | 2.114674 | hg-manifest -| Algorithm | Speed(sec) | Compression Ratio | -| ------------- |:-------------:| ------------------:| -| nodict | 0.000005 | 1.866385 | -| random | 1.035220 | 2.309485 | -| cover | 930.480173 | 2.582597 | -| legacy | 8.916513 | 2.506775 | -| fastCover | 116.871089 | 2.525689 | +| Algorithm | Speed(sec) | Compression Ratio | +| ----------------- |:-------------:| ------------------:| +| nodict | 0.000019 | 1.866385 | +| random | 1.083536 | 2.309485 | +| cover | 928.894887 | 2.582597 | +| legacy | 9.110371 | 2.506775 | +| fastCover(opt) | 116.508270 | 2.525689 | +| fastCover(k=200) | 12.176555 | 2.472221 | diff --git a/contrib/experimental_dict_builders/benchmarkDictBuilder/benchmark.c b/contrib/experimental_dict_builders/benchmarkDictBuilder/benchmark.c index 865ecb34d..62135436b 100644 --- a/contrib/experimental_dict_builders/benchmarkDictBuilder/benchmark.c +++ b/contrib/experimental_dict_builders/benchmarkDictBuilder/benchmark.c @@ -373,7 +373,8 @@ int main(int argCount, const char* argv[]) } } - /* for fastCover */ + + /* for fastCover (optimizing k) */ { ZDICT_fastCover_params_t fastParam; memset(&fastParam, 0, sizeof(fastParam)); @@ -390,6 +391,25 @@ int main(int argCount, const char* argv[]) } } + /* for fastCover (with k provided) */ + { + ZDICT_fastCover_params_t fastParam; + memset(&fastParam, 0, sizeof(fastParam)); + fastParam.zParams = zParams; + fastParam.splitPoint = 1.0; + fastParam.d = 8; + fastParam.f = 23; + fastParam.k = 200; + fastParam.steps = 40; + fastParam.nbThreads = 1; + const int fastOptResult = benchmarkDictBuilder(srcInfo, maxDictSize, NULL, NULL, NULL, &fastParam); + if(fastOptResult) { + result = 1; + goto _cleanup; + } + } + + /* Free allocated memory */ _cleanup: UTIL_freeFileList(extendedFileList, fileNamesBuf); From 2333ecb173077edaf34f032baadfcc63531928c1 Mon Sep 17 00:00:00 2001 From: Jennifer Liu Date: Wed, 25 Jul 2018 18:10:09 -0700 Subject: [PATCH 076/372] Allow d=6 --- .../fastCover/README.md | 2 +- .../fastCover/fastCover.c | 27 +++++++++++++------ .../fastCover/test.sh | 3 ++- 3 files changed, 22 insertions(+), 10 deletions(-) diff --git a/contrib/experimental_dict_builders/fastCover/README.md b/contrib/experimental_dict_builders/fastCover/README.md index 088e38be7..66e00ee04 100644 --- a/contrib/experimental_dict_builders/fastCover/README.md +++ b/contrib/experimental_dict_builders/fastCover/README.md @@ -6,7 +6,7 @@ Output Dictionary (out=dictName): if not provided, default to fastCoverDict Dictionary ID (dictID=#): nonnegative number; if not provided, default to 0 Maximum Dictionary Size (maxdict=#): positive number; in bytes, if not provided, default to 110KB Size of Selected Segment (k=#): positive number; in bytes; if not provided, default to 200 -Size of Dmer (d=#): positive number; in bytes; if not provided, default to 8 +Size of Dmer (d=#): either 6 or 8; if not provided, default to 8 Number of steps (steps=#): positive number, if not provided, default to 32 Percentage of samples used for training(split=#): positive number; if not provided, default to 100 diff --git a/contrib/experimental_dict_builders/fastCover/fastCover.c b/contrib/experimental_dict_builders/fastCover/fastCover.c index 32a15a4bf..abd592cd8 100644 --- a/contrib/experimental_dict_builders/fastCover/fastCover.c +++ b/contrib/experimental_dict_builders/fastCover/fastCover.c @@ -50,14 +50,21 @@ static clock_t g_time = 0; /*-************************************* * Hash Function ***************************************/ +static const U64 prime6bytes = 227718039650203ULL; +static size_t ZSTD_hash6(U64 u, U32 h) { return (size_t)(((u << (64-48)) * prime6bytes) >> (64-h)) ; } +static size_t ZSTD_hash6Ptr(const void* p, U32 h) { return ZSTD_hash6(MEM_readLE64(p), h); } + static const U64 prime8bytes = 0xCF1BBCDCB7A56463ULL; static size_t ZSTD_hash8(U64 u, U32 h) { return (size_t)(((u) * prime8bytes) >> (64-h)) ; } static size_t ZSTD_hash8Ptr(const void* p, U32 h) { return ZSTD_hash8(MEM_readLE64(p), h); } /** - * Hash the 8-byte value pointed to by p and mod 2^f + * Hash the d-byte value pointed to by p and mod 2^f */ -static size_t FASTCOVER_hash8PtrToIndex(const void* p, U32 h) { +static size_t FASTCOVER_hashPtrToIndex(const void* p, U32 h, unsigned d) { + if (d == 6) { + return ZSTD_hash6Ptr(p, h) & ((1 << h) - 1); + } return ZSTD_hash8Ptr(p, h) & ((1 << h) - 1); } @@ -138,7 +145,7 @@ static FASTCOVER_segment_t FASTCOVER_selectSegment(const FASTCOVER_ctx_t *ctx, */ while (activeSegment.end < end) { /* Get hash value of current dmer */ - const size_t index = FASTCOVER_hash8PtrToIndex(ctx->samples + activeSegment.end, parameters.f); + const size_t index = FASTCOVER_hashPtrToIndex(ctx->samples + activeSegment.end, parameters.f, ctx->d); /* Add frequency of this index to score */ activeSegment.score += freqs[index]; /* Increment end of segment */ @@ -146,7 +153,7 @@ static FASTCOVER_segment_t FASTCOVER_selectSegment(const FASTCOVER_ctx_t *ctx, /* If the window is now too large, drop the first position */ if (activeSegment.end - activeSegment.begin == dmersInK + 1) { /* Get hash value of the dmer to be eliminated from active segment */ - const size_t delIndex = FASTCOVER_hash8PtrToIndex(ctx->samples + activeSegment.begin, parameters.f); + const size_t delIndex = FASTCOVER_hashPtrToIndex(ctx->samples + activeSegment.begin, parameters.f, ctx->d); /* Subtract frequency of this index from score */ activeSegment.score -= freqs[delIndex]; /* Increment start of segment */ @@ -163,7 +170,7 @@ static FASTCOVER_segment_t FASTCOVER_selectSegment(const FASTCOVER_ctx_t *ctx, U32 newEnd = bestSegment.begin; U32 pos; for (pos = bestSegment.begin; pos != bestSegment.end; ++pos) { - const size_t index = FASTCOVER_hash8PtrToIndex(ctx->samples + pos, parameters.f); + const size_t index = FASTCOVER_hashPtrToIndex(ctx->samples + pos, parameters.f, ctx->d); U32 freq = freqs[index]; if (freq != 0) { newBegin = MIN(newBegin, pos); @@ -177,7 +184,7 @@ static FASTCOVER_segment_t FASTCOVER_selectSegment(const FASTCOVER_ctx_t *ctx, /* Half the frequency of hash value of each dmer covered by the chosen segment. */ U32 pos; for (pos = bestSegment.begin; pos != bestSegment.end; ++pos) { - const size_t i = FASTCOVER_hash8PtrToIndex(ctx->samples + pos, parameters.f); + const size_t i = FASTCOVER_hashPtrToIndex(ctx->samples + pos, parameters.f, ctx->d); freqs[i] = freqs[i]/2; } } @@ -194,6 +201,10 @@ static int FASTCOVER_checkParameters(ZDICT_fastCover_params_t parameters, if (parameters.d == 0 || parameters.k == 0 || parameters.f == 0) { return 0; } + /* d has to be 6 or 8 */ + if (parameters.d != 6 && parameters.d != 8) { + return 0; + } /* 0 < f <= FASTCOVER_MAX_F */ if (parameters.f > FASTCOVER_MAX_F) { return 0; @@ -244,7 +255,7 @@ static void FASTCOVER_getFrequency(U32 *freqs, unsigned f, FASTCOVER_ctx_t *ctx) size_t currSampleEnd = ctx->offsets[i+1]; start = currSampleStart; while (start + f < currSampleEnd) { - const size_t dmerIndex = FASTCOVER_hash8PtrToIndex(ctx->samples + start, f); + const size_t dmerIndex = FASTCOVER_hashPtrToIndex(ctx->samples + start, f, ctx->d); /* if no dmer with same hash value has been seen in current sample */ if (inCurrSample[dmerIndex] == 0) { inCurrSample[dmerIndex]++; @@ -615,7 +626,7 @@ ZDICTLIB_API size_t ZDICT_optimizeTrainFromBuffer_fastCover( const unsigned nbThreads = parameters->nbThreads; const double splitPoint = parameters->splitPoint <= 0.0 ? DEFAULT_SPLITPOINT : parameters->splitPoint; - const unsigned kMinD = parameters->d == 0 ? 8 : parameters->d; + const unsigned kMinD = parameters->d == 0 ? 6 : parameters->d; const unsigned kMaxD = parameters->d == 0 ? 8 : parameters->d; const unsigned kMinK = parameters->k == 0 ? 50 : parameters->k; const unsigned kMaxK = parameters->k == 0 ? 2000 : parameters->k; diff --git a/contrib/experimental_dict_builders/fastCover/test.sh b/contrib/experimental_dict_builders/fastCover/test.sh index b5570fef1..91d4f4923 100644 --- a/contrib/experimental_dict_builders/fastCover/test.sh +++ b/contrib/experimental_dict_builders/fastCover/test.sh @@ -11,4 +11,5 @@ echo "Removing dict1 dict2 dict3" rm -f dict1 dict2 dict3 echo "Testing with invalid parameters, should fail" -! ./main r=10 +! ./main in=../../../lib/common r=10 +! ./main in=../../../lib/common d=10 From 3b163e0b5b5f9eec427b87001483c3b627c95a8f Mon Sep 17 00:00:00 2001 From: Jennifer Liu Date: Thu, 26 Jul 2018 13:53:13 -0700 Subject: [PATCH 077/372] Add array to keep track of frequency within active segment, fix malloc bug, update benchmarking result --- .../benchmarkDictBuilder/README.md | 60 ++++++++-------- .../fastCover/fastCover.c | 69 +++++++++++-------- .../fastCover/main.c | 2 +- .../randomDictBuilder/main.c | 2 +- 4 files changed, 75 insertions(+), 58 deletions(-) diff --git a/contrib/experimental_dict_builders/benchmarkDictBuilder/README.md b/contrib/experimental_dict_builders/benchmarkDictBuilder/README.md index 478d8793e..07d65b08c 100644 --- a/contrib/experimental_dict_builders/benchmarkDictBuilder/README.md +++ b/contrib/experimental_dict_builders/benchmarkDictBuilder/README.md @@ -14,42 +14,46 @@ make ARG="in=../../../lib/dictBuilder in=../../../lib/compress" ###Benchmarking Result: +d=8 +f=23 +freq[i] = 0 when dmer added to best segment + github: | Algorithm | Speed(sec) | Compression Ratio | -| ------------------|:-------------:| ------------------:| -| nodict | 0.000004 | 2.999642 | -| random | 0.148247 | 8.786957 | -| cover | 56.331553 | 10.641263 | -| legacy | 0.917595 | 8.989482 | -| fastCover(opt) | 13.169979 | 10.215174 | -| fastCover(k=200) | 2.692406 | 8.657219 | +| ----------------- | ------------- | ------------------ | +| nodict | 0.000007 | 2.999642 | +| random | 0.150258 | 8.786957 | +| cover | 60.388853 | 10.641263 | +| legacy | 0.965050 | 8.989482 | +| fastCover(opt) | 84.968131 | 10.614747 | +| fastCover(k=200) | 6.465490 | 9.484150 | hg-commands | Algorithm | Speed(sec) | Compression Ratio | -| ----------------- |:-------------:| ------------------:| -| nodict | 0.000007 | 2.425291 | -| random | 0.093990 | 3.489515 | -| cover | 58.602385 | 4.131136 | -| legacy | 0.865683 | 3.911896 | -| fastCover(opt) | 9.404134 | 3.977229 | -| fastCover(k=200) | 1.037434 | 3.810326 | +| ----------------- | ------------- | ------------------ | +| nodict | 0.000005 | 2.425291 | +| random | 0.084348 | 3.489515 | +| cover | 60.144894 | 4.131136 | +| legacy | 0.831981 | 3.911896 | +| fastCover(opt) | 59.030437 | 4.157595 | +| fastCover(k=200) | 3.702932 | 4.134222 | hg-changelog | Algorithm | Speed(sec) | Compression Ratio | -| ----------------- |:-------------:| ------------------:| -| nodict | 0.000022 | 1.377613 | -| random | 0.551539 | 2.096785 | -| cover | 221.370056 | 2.188654 | -| legacy | 2.405923 | 2.058273 | -| fastCover(opt) | 49.526246 | 2.124185 | -| fastCover(k=200) | 9.746872 | 2.114674 | +| ----------------- | ------------- | ------------------ | +| nodict | 0.000004 | 1.377613 | +| random | 0.555964 | 2.096785 | +| cover | 214.423753 | 2.188654 | +| legacy | 2.180249 | 2.058273 | +| fastCover(opt) | 102.261452 | 2.180347 | +| fastCover(k=200) | 11.81039 | 2.170673 | hg-manifest | Algorithm | Speed(sec) | Compression Ratio | -| ----------------- |:-------------:| ------------------:| -| nodict | 0.000019 | 1.866385 | -| random | 1.083536 | 2.309485 | -| cover | 928.894887 | 2.582597 | -| legacy | 9.110371 | 2.506775 | -| fastCover(opt) | 116.508270 | 2.525689 | -| fastCover(k=200) | 12.176555 | 2.472221 | +| ----------------- | ------------- | ------------------ | +| nodict | 0.000006 | 1.866385 | +| random | 1.063974 | 2.309485 | +| cover | 909.101849 | 2.582597 | +| legacy | 8.706580 | 2.506775 | +| fastCover(opt) | 188.598079 | 2.596761 | +| fastCover(k=200) | 13.392734 | 2.592985 | diff --git a/contrib/experimental_dict_builders/fastCover/fastCover.c b/contrib/experimental_dict_builders/fastCover/fastCover.c index abd592cd8..6f990e0c2 100644 --- a/contrib/experimental_dict_builders/fastCover/fastCover.c +++ b/contrib/experimental_dict_builders/fastCover/fastCover.c @@ -48,7 +48,7 @@ static clock_t g_time = 0; /*-************************************* -* Hash Function +* Hash Functions ***************************************/ static const U64 prime6bytes = 227718039650203ULL; static size_t ZSTD_hash6(U64 u, U32 h) { return (size_t)(((u << (64-48)) * prime6bytes) >> (64-h)) ; } @@ -58,6 +58,7 @@ static const U64 prime8bytes = 0xCF1BBCDCB7A56463ULL; static size_t ZSTD_hash8(U64 u, U32 h) { return (size_t)(((u) * prime8bytes) >> (64-h)) ; } static size_t ZSTD_hash8Ptr(const void* p, U32 h) { return ZSTD_hash8(MEM_readLE64(p), h); } + /** * Hash the d-byte value pointed to by p and mod 2^f */ @@ -140,29 +141,41 @@ static FASTCOVER_segment_t FASTCOVER_selectSegment(const FASTCOVER_ctx_t *ctx, activeSegment.begin = begin; activeSegment.end = begin; activeSegment.score = 0; - /* Slide the activeSegment through the whole epoch. - * Save the best segment in bestSegment. - */ - while (activeSegment.end < end) { - /* Get hash value of current dmer */ - const size_t index = FASTCOVER_hashPtrToIndex(ctx->samples + activeSegment.end, parameters.f, ctx->d); - /* Add frequency of this index to score */ - activeSegment.score += freqs[index]; - /* Increment end of segment */ - activeSegment.end += 1; - /* If the window is now too large, drop the first position */ - if (activeSegment.end - activeSegment.begin == dmersInK + 1) { - /* Get hash value of the dmer to be eliminated from active segment */ - const size_t delIndex = FASTCOVER_hashPtrToIndex(ctx->samples + activeSegment.begin, parameters.f, ctx->d); - /* Subtract frequency of this index from score */ - activeSegment.score -= freqs[delIndex]; - /* Increment start of segment */ - activeSegment.begin += 1; - } - /* If this segment is the best so far save it */ - if (activeSegment.score > bestSegment.score) { - bestSegment = activeSegment; + { + /* Keep track of number of times an index has been seen in current segment */ + U16* currfreqs =(U16 *)malloc((1 << parameters.f) * sizeof(U16)); + memset(currfreqs, 0, (1 << parameters.f) * sizeof(*currfreqs)); + /* Slide the activeSegment through the whole epoch. + * Save the best segment in bestSegment. + */ + while (activeSegment.end < end) { + /* Get hash value of current dmer */ + const size_t index = FASTCOVER_hashPtrToIndex(ctx->samples + activeSegment.end, parameters.f, ctx->d); + /* Add frequency of this index to score if this is the first occurence of index in active segment */ + if (currfreqs[index] == 0) { + activeSegment.score += freqs[index]; + } + currfreqs[index] += 1; + /* Increment end of segment */ + activeSegment.end += 1; + /* If the window is now too large, drop the first position */ + if (activeSegment.end - activeSegment.begin == dmersInK + 1) { + /* Get hash value of the dmer to be eliminated from active segment */ + const size_t delIndex = FASTCOVER_hashPtrToIndex(ctx->samples + activeSegment.begin, parameters.f, ctx->d); + currfreqs[delIndex] -= 1; + /* Subtract frequency of this index from score if this is the last occurrence of this index in active segment */ + if (currfreqs[delIndex] == 0) { + activeSegment.score -= freqs[delIndex]; + } + /* Increment start of segment */ + activeSegment.begin += 1; + } + /* If this segment is the best so far save it */ + if (activeSegment.score > bestSegment.score) { + bestSegment = activeSegment; + } } + free(currfreqs); } { /* Trim off the zero frequency head and tail from the segment. */ @@ -185,7 +198,7 @@ static FASTCOVER_segment_t FASTCOVER_selectSegment(const FASTCOVER_ctx_t *ctx, U32 pos; for (pos = bestSegment.begin; pos != bestSegment.end; ++pos) { const size_t i = FASTCOVER_hashPtrToIndex(ctx->samples + pos, parameters.f, ctx->d); - freqs[i] = freqs[i]/2; + freqs[i] = 0; } } return bestSegment; @@ -245,12 +258,12 @@ static void FASTCOVER_ctx_destroy(FASTCOVER_ctx_t *ctx) { /** * Calculate for frequency of hash value of each dmer in ctx->samples */ -static void FASTCOVER_getFrequency(U32 *freqs, unsigned f, FASTCOVER_ctx_t *ctx){ +static void FASTCOVER_computeFrequency(U32 *freqs, unsigned f, FASTCOVER_ctx_t *ctx){ /* inCurrSample keeps track of this hash value has already be seen in previous dmers in the same sample*/ - size_t* inCurrSample = (size_t *)malloc((1<nbTrainSamples; i++) { - memset(inCurrSample, 0, (1 << f)); /* Reset inCurrSample for each sample */ + memset(inCurrSample, 0, (1 << f) * sizeof(*inCurrSample)); /* Reset inCurrSample for each sample */ size_t currSampleStart = ctx->offsets[i]; size_t currSampleEnd = ctx->offsets[i+1]; start = currSampleStart; @@ -338,7 +351,7 @@ static int FASTCOVER_ctx_init(FASTCOVER_ctx_t *ctx, const void *samplesBuffer, memset(ctx->freqs, 0, (1 << f) * sizeof(U32)); DISPLAYLEVEL(2, "Computing frequencies\n"); - FASTCOVER_getFrequency(ctx->freqs, f, ctx); + FASTCOVER_computeFrequency(ctx->freqs, f, ctx); return 1; } diff --git a/contrib/experimental_dict_builders/fastCover/main.c b/contrib/experimental_dict_builders/fastCover/main.c index 260eeb281..f286b0506 100644 --- a/contrib/experimental_dict_builders/fastCover/main.c +++ b/contrib/experimental_dict_builders/fastCover/main.c @@ -165,7 +165,7 @@ int main(int argCount, const char* argv[]) params.splitPoint = (double)split/100; /* Build dictionary */ - sampleInfo* info= getSampleInfo(filenameTable, + sampleInfo* info = getSampleInfo(filenameTable, filenameIdx, blockSize, maxDictSize, zParams.notificationLevel); operationResult = FASTCOVER_trainFromFiles(outputFile, info, maxDictSize, ¶ms); diff --git a/contrib/experimental_dict_builders/randomDictBuilder/main.c b/contrib/experimental_dict_builders/randomDictBuilder/main.c index 3f3a6ca70..3ad885746 100644 --- a/contrib/experimental_dict_builders/randomDictBuilder/main.c +++ b/contrib/experimental_dict_builders/randomDictBuilder/main.c @@ -149,7 +149,7 @@ int main(int argCount, const char* argv[]) params.zParams = zParams; params.k = k; - sampleInfo* info= getSampleInfo(filenameTable, + sampleInfo* info = getSampleInfo(filenameTable, filenameIdx, blockSize, maxDictSize, zParams.notificationLevel); operationResult = RANDOM_trainFromFiles(outputFile, info, maxDictSize, ¶ms); From 09ccd977c355c07a469a295837397abe28b6fdb2 Mon Sep 17 00:00:00 2001 From: George Lu Date: Thu, 26 Jul 2018 15:17:58 -0700 Subject: [PATCH 078/372] no zero --- programs/bench.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/programs/bench.c b/programs/bench.c index a54168c42..76d1ff6dd 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -549,7 +549,8 @@ static BMK_return_t BMK_benchMemAdvancedNoAlloc( double const compressionSpeed = ((double)srcSize / intermediateResultCompress.result.result.nanoSecPerRun) * 1000; int const cSpeedAccuracy = (compressionSpeed < 10.) ? 2 : 1; results.result.cSpeed = compressionSpeed * 1000000; - results.result.cSize = intermediateResultCompress.result.result.sumOfReturn; + cSize = intermediateResultCompress.result.result.sumOfReturn; + results.result.cSize = cSize; ratio = (double)srcSize / results.result.cSize; markNb = (markNb+1) % NB_MARKS; DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.*f),%6.*f MB/s\r", From 3d7941ce41d33bbbedb15fa9794c9fbcb1713384 Mon Sep 17 00:00:00 2001 From: Jennifer Liu Date: Thu, 26 Jul 2018 16:24:13 -0700 Subject: [PATCH 079/372] Benchmark different f values --- .../benchmarkDictBuilder/README.md | 131 +++++++++++++----- .../benchmarkDictBuilder/benchmark.c | 104 +++++++------- 2 files changed, 152 insertions(+), 83 deletions(-) diff --git a/contrib/experimental_dict_builders/benchmarkDictBuilder/README.md b/contrib/experimental_dict_builders/benchmarkDictBuilder/README.md index 07d65b08c..1ee4b19ba 100644 --- a/contrib/experimental_dict_builders/benchmarkDictBuilder/README.md +++ b/contrib/experimental_dict_builders/benchmarkDictBuilder/README.md @@ -14,46 +14,107 @@ make ARG="in=../../../lib/dictBuilder in=../../../lib/compress" ###Benchmarking Result: -d=8 -f=23 -freq[i] = 0 when dmer added to best segment +For every f value for fast, the first one is optimize and the second one has k=200 github: -| Algorithm | Speed(sec) | Compression Ratio | -| ----------------- | ------------- | ------------------ | -| nodict | 0.000007 | 2.999642 | -| random | 0.150258 | 8.786957 | -| cover | 60.388853 | 10.641263 | -| legacy | 0.965050 | 8.989482 | -| fastCover(opt) | 84.968131 | 10.614747 | -| fastCover(k=200) | 6.465490 | 9.484150 | +NODICT 0.000023 2.999642 +RANDOM 0.149020 8.786957 +LEGACY 0.854277 8.989482 +FAST15 8.764078 10.609015 +FAST15 0.232610 9.135669 +FAST16 9.597777 10.474574 +FAST16 0.243698 9.346482 +FAST17 9.385449 10.611737 +FAST17 0.268376 9.605798 +FAST18 9.988885 10.626382 +FAST18 0.311769 9.130565 +FAST19 10.737259 10.411729 +FAST19 0.331885 9.271814 +FAST20 10.479782 10.388895 +FAST20 0.498416 9.194115 +FAST21 21.189883 10.376394 +FAST21 1.098532 9.244456 +FAST22 39.849935 10.432555 +FAST22 2.590561 9.410930 +FAST23 75.832399 10.614747 +FAST23 6.108487 9.484150 +FAST24 139.782714 10.611753 +FAST24 13.029406 9.379030 +COVER 55.118542 10.641263 hg-commands -| Algorithm | Speed(sec) | Compression Ratio | -| ----------------- | ------------- | ------------------ | -| nodict | 0.000005 | 2.425291 | -| random | 0.084348 | 3.489515 | -| cover | 60.144894 | 4.131136 | -| legacy | 0.831981 | 3.911896 | -| fastCover(opt) | 59.030437 | 4.157595 | -| fastCover(k=200) | 3.702932 | 4.134222 | +NODICT 0.000012 2.425291 +RANDOM 0.083071 3.489515 +LEGACY 0.835195 3.911896 +FAST15 0.163980 3.808375 +FAST16 6.373850 4.010783 +FAST16 0.160299 3.966604 +FAST17 6.668799 4.091602 +FAST17 0.172480 4.062773 +FAST18 6.266105 4.130824 +FAST18 0.171554 4.094666 +FAST19 6.869651 4.158180 +FAST19 0.209468 4.111289 +FAST20 8.267766 4.149707 +FAST20 0.331680 4.119873 +FAST21 18.824296 4.171784 +FAST21 0.783961 4.120884 +FAST22 33.321252 4.152035 +FAST22 1.854215 4.126626 +FAST23 60.775388 4.157595 +FAST23 4.040395 4.134222 +FAST24 110.910038 4.163091 +FAST24 8.505828 4.143533 +COVER 61.654796 4.131136 hg-changelog -| Algorithm | Speed(sec) | Compression Ratio | -| ----------------- | ------------- | ------------------ | -| nodict | 0.000004 | 1.377613 | -| random | 0.555964 | 2.096785 | -| cover | 214.423753 | 2.188654 | -| legacy | 2.180249 | 2.058273 | -| fastCover(opt) | 102.261452 | 2.180347 | -| fastCover(k=200) | 11.81039 | 2.170673 | +NODICT 0.000004 1.377613 +RANDOM 0.582067 2.096785 +LEGACY 2.739515 2.058273 +FAST15 35.682665 2.127596 +FAST15 0.931621 2.115299 +FAST16 36.557988 2.141787 +FAST16 1.008155 2.136080 +FAST17 36.272242 2.155332 +FAST17 0.906803 2.154596 +FAST18 35.542043 2.171997 +FAST18 1.063101 2.167723 +FAST19 37.756934 2.180893 +FAST19 1.257291 2.173768 +FAST20 40.273755 2.179442 +FAST20 1.630522 2.170072 +FAST21 54.606548 2.181400 +FAST21 2.321266 2.171643 +FAST22 72.454066 2.178774 +FAST22 5.092888 2.168885 +FAST23 106.753208 2.180347 +FAST23 14.722222 2.170673 +FAST24 171.083201 2.183426 +FAST24 27.575575 2.170623 +COVER 227.219660 2.188654 hg-manifest -| Algorithm | Speed(sec) | Compression Ratio | -| ----------------- | ------------- | ------------------ | -| nodict | 0.000006 | 1.866385 | -| random | 1.063974 | 2.309485 | -| cover | 909.101849 | 2.582597 | -| legacy | 8.706580 | 2.506775 | -| fastCover(opt) | 188.598079 | 2.596761 | -| fastCover(k=200) | 13.392734 | 2.592985 | +NODICT 0.000007 1.866385 +RANDOM 1.086571 2.309485 +LEGACY 9.567507 2.506775 +FAST15 77.811380 2.380461 +FAST15 1.969718 2.317727 +FAST16 75.789019 2.469144 +FAST16 2.051283 2.375815 +FAST17 79.659040 2.539069 +FAST17 1.995394 2.501047 +FAST18 76.281105 2.578095 +FAST18 2.059272 2.564840 +FAST19 79.395382 2.590433 +FAST19 2.354158 2.591024 +FAST20 87.937568 2.597813 +FAST20 2.922189 2.597104 +FAST21 121.760549 2.598408 +FAST21 4.798981 2.600269 +FAST22 155.878461 2.594560 +FAST22 8.151807 2.601047 +FAST23 194.238003 2.596761 +FAST23 15.160578 2.592985 +FAST24 267.425904 2.597657 +FAST24 29.513286 2.600363 +COVER 930.675322 2.582597 diff --git a/contrib/experimental_dict_builders/benchmarkDictBuilder/benchmark.c b/contrib/experimental_dict_builders/benchmarkDictBuilder/benchmark.c index 62135436b..9feaae592 100644 --- a/contrib/experimental_dict_builders/benchmarkDictBuilder/benchmark.c +++ b/contrib/experimental_dict_builders/benchmarkDictBuilder/benchmark.c @@ -340,12 +340,67 @@ int main(int argCount, const char* argv[]) randomParam.zParams = zParams; randomParam.k = k; const int randomResult = benchmarkDictBuilder(srcInfo, maxDictSize, &randomParam, NULL, NULL, NULL); + DISPLAYLEVEL(2, "k=%u\n", randomParam.k); if(randomResult) { result = 1; goto _cleanup; } } + /* for legacy */ + { + ZDICT_legacy_params_t legacyParam; + legacyParam.zParams = zParams; + legacyParam.selectivityLevel = 9; + const int legacyResult = benchmarkDictBuilder(srcInfo, maxDictSize, NULL, NULL, &legacyParam, NULL); + DISPLAYLEVEL(2, "selectivityLevel=%u\n", legacyParam.selectivityLevel); + if(legacyResult) { + result = 1; + goto _cleanup; + } + } + + /* for fastCover */ + for (unsigned f = 15; f < 25; f++){ + DISPLAYLEVEL(2, "current f is %u\n", f); + /* for fastCover (optimizing k) */ + { + ZDICT_fastCover_params_t fastParam; + memset(&fastParam, 0, sizeof(fastParam)); + fastParam.zParams = zParams; + fastParam.splitPoint = 1.0; + fastParam.d = 8; + fastParam.f = f; + fastParam.steps = 40; + fastParam.nbThreads = 1; + const int fastOptResult = benchmarkDictBuilder(srcInfo, maxDictSize, NULL, NULL, NULL, &fastParam); + DISPLAYLEVEL(2, "k=%u\nd=%u\nf=%u\nsteps=%u\nsplit=%u\n", fastParam.k, fastParam.d, fastParam.f, fastParam.steps, (unsigned)(fastParam.splitPoint * 100)); + if(fastOptResult) { + result = 1; + goto _cleanup; + } + } + + /* for fastCover (with k provided) */ + { + ZDICT_fastCover_params_t fastParam; + memset(&fastParam, 0, sizeof(fastParam)); + fastParam.zParams = zParams; + fastParam.splitPoint = 1.0; + fastParam.d = 8; + fastParam.f = f; + fastParam.k = 200; + fastParam.steps = 40; + fastParam.nbThreads = 1; + const int fastOptResult = benchmarkDictBuilder(srcInfo, maxDictSize, NULL, NULL, NULL, &fastParam); + DISPLAYLEVEL(2, "k=%u\nd=%u\nf=%u\nsteps=%u\nsplit=%u\n", fastParam.k, fastParam.d, fastParam.f, fastParam.steps, (unsigned)(fastParam.splitPoint * 100)); + if(fastOptResult) { + result = 1; + goto _cleanup; + } + } + } + /* for cover */ { ZDICT_cover_params_t coverParam; @@ -355,60 +410,13 @@ int main(int argCount, const char* argv[]) coverParam.steps = 40; coverParam.nbThreads = 1; const int coverOptResult = benchmarkDictBuilder(srcInfo, maxDictSize, NULL, &coverParam, NULL, NULL); + DISPLAYLEVEL(2, "k=%u\nd=%u\nsteps=%u\nsplit=%u\n", coverParam.k, coverParam.d, coverParam.steps, (unsigned)(coverParam.splitPoint * 100)); if(coverOptResult) { result = 1; goto _cleanup; } } - /* for legacy */ - { - ZDICT_legacy_params_t legacyParam; - legacyParam.zParams = zParams; - legacyParam.selectivityLevel = 9; - const int legacyResult = benchmarkDictBuilder(srcInfo, maxDictSize, NULL, NULL, &legacyParam, NULL); - if(legacyResult) { - result = 1; - goto _cleanup; - } - } - - - /* for fastCover (optimizing k) */ - { - ZDICT_fastCover_params_t fastParam; - memset(&fastParam, 0, sizeof(fastParam)); - fastParam.zParams = zParams; - fastParam.splitPoint = 1.0; - fastParam.d = 8; - fastParam.f = 23; - fastParam.steps = 40; - fastParam.nbThreads = 1; - const int fastOptResult = benchmarkDictBuilder(srcInfo, maxDictSize, NULL, NULL, NULL, &fastParam); - if(fastOptResult) { - result = 1; - goto _cleanup; - } - } - - /* for fastCover (with k provided) */ - { - ZDICT_fastCover_params_t fastParam; - memset(&fastParam, 0, sizeof(fastParam)); - fastParam.zParams = zParams; - fastParam.splitPoint = 1.0; - fastParam.d = 8; - fastParam.f = 23; - fastParam.k = 200; - fastParam.steps = 40; - fastParam.nbThreads = 1; - const int fastOptResult = benchmarkDictBuilder(srcInfo, maxDictSize, NULL, NULL, NULL, &fastParam); - if(fastOptResult) { - result = 1; - goto _cleanup; - } - } - /* Free allocated memory */ _cleanup: From 759c543312fd722c6f351513411d6d57742c7e4e Mon Sep 17 00:00:00 2001 From: Jennifer Liu Date: Thu, 26 Jul 2018 19:03:01 -0700 Subject: [PATCH 080/372] Rerun cover and fastCover with optimized values --- .../benchmarkDictBuilder/README.md | 197 +++++++++--------- .../benchmarkDictBuilder/benchmark.c | 109 ++++++---- .../fastCover/fastCover.c | 2 +- 3 files changed, 169 insertions(+), 139 deletions(-) diff --git a/contrib/experimental_dict_builders/benchmarkDictBuilder/README.md b/contrib/experimental_dict_builders/benchmarkDictBuilder/README.md index 1ee4b19ba..04866b7e6 100644 --- a/contrib/experimental_dict_builders/benchmarkDictBuilder/README.md +++ b/contrib/experimental_dict_builders/benchmarkDictBuilder/README.md @@ -13,108 +13,113 @@ Benchmark given input files: make ARG= followed by permitted arguments make ARG="in=../../../lib/dictBuilder in=../../../lib/compress" ###Benchmarking Result: - -For every f value for fast, the first one is optimize and the second one has k=200 +First Cover is optimize cover, second Cover uses optimized d and k from first one. +For every f value of fastCover, the first one is optimize fastCover and the second one uses optimized d and k from first one. github: -NODICT 0.000023 2.999642 -RANDOM 0.149020 8.786957 -LEGACY 0.854277 8.989482 -FAST15 8.764078 10.609015 -FAST15 0.232610 9.135669 -FAST16 9.597777 10.474574 -FAST16 0.243698 9.346482 -FAST17 9.385449 10.611737 -FAST17 0.268376 9.605798 -FAST18 9.988885 10.626382 -FAST18 0.311769 9.130565 -FAST19 10.737259 10.411729 -FAST19 0.331885 9.271814 -FAST20 10.479782 10.388895 -FAST20 0.498416 9.194115 -FAST21 21.189883 10.376394 -FAST21 1.098532 9.244456 -FAST22 39.849935 10.432555 -FAST22 2.590561 9.410930 -FAST23 75.832399 10.614747 -FAST23 6.108487 9.484150 -FAST24 139.782714 10.611753 -FAST24 13.029406 9.379030 -COVER 55.118542 10.641263 +NODICT 0.000004 2.999642 +RANDOM 0.146096 8.786957 +LEGACY 0.956888 8.989482 +COVER 56.596152 10.641263 +COVER 4.937047 10.641263 +FAST15 17.722269 10.586461 +FAST15 0.239135 10.586461 +FAST16 18.276179 10.492503 +FAST16 0.265285 10.492503 +FAST17 18.077916 10.611737 +FAST17 0.236573 10.611737 +FAST18 19.510150 10.621586 +FAST18 0.278683 10.621586 +FAST19 18.794350 10.629626 +FAST19 0.307943 10.629626 +FAST20 19.671099 10.610308 +FAST20 0.428814 10.610308 +FAST21 36.527238 10.625733 +FAST21 0.716384 10.625733 +FAST22 83.803521 10.625281 +FAST22 1.290246 10.625281 +FAST23 158.287924 10.602342 +FAST23 3.084848 10.602342 +FAST24 283.630941 10.603379 +FAST24 8.088933 10.603379 hg-commands -NODICT 0.000012 2.425291 -RANDOM 0.083071 3.489515 -LEGACY 0.835195 3.911896 -FAST15 0.163980 3.808375 -FAST16 6.373850 4.010783 -FAST16 0.160299 3.966604 -FAST17 6.668799 4.091602 -FAST17 0.172480 4.062773 -FAST18 6.266105 4.130824 -FAST18 0.171554 4.094666 -FAST19 6.869651 4.158180 -FAST19 0.209468 4.111289 -FAST20 8.267766 4.149707 -FAST20 0.331680 4.119873 -FAST21 18.824296 4.171784 -FAST21 0.783961 4.120884 -FAST22 33.321252 4.152035 -FAST22 1.854215 4.126626 -FAST23 60.775388 4.157595 -FAST23 4.040395 4.134222 -FAST24 110.910038 4.163091 -FAST24 8.505828 4.143533 -COVER 61.654796 4.131136 +NODICT 0.000007 2.425291 +RANDOM 0.084010 3.489515 +LEGACY 0.926763 3.911896 +COVER 62.036915 4.131136 +COVER 2.194398 4.131136 +FAST15 12.169025 3.903719 +FAST15 0.156552 3.903719 +FAST16 11.886255 4.005077 +FAST16 0.155506 4.005077 +FAST17 11.886955 4.097811 +FAST17 0.176327 4.097811 +FAST18 12.544698 4.136081 +FAST18 0.171796 4.136081 +FAST19 12.920868 4.166021 +FAST19 0.207029 4.166021 +FAST20 15.771429 4.163740 +FAST20 0.258685 4.163740 +FAST21 33.165829 4.157057 +FAST21 0.663088 4.157057 +FAST22 68.779201 4.158195 +FAST22 1.568439 4.158195 +FAST23 121.921931 4.161450 +FAST23 2.498972 4.161450 +FAST24 221.990451 4.159658 +FAST24 5.793594 4.159658 hg-changelog NODICT 0.000004 1.377613 -RANDOM 0.582067 2.096785 -LEGACY 2.739515 2.058273 -FAST15 35.682665 2.127596 -FAST15 0.931621 2.115299 -FAST16 36.557988 2.141787 -FAST16 1.008155 2.136080 -FAST17 36.272242 2.155332 -FAST17 0.906803 2.154596 -FAST18 35.542043 2.171997 -FAST18 1.063101 2.167723 -FAST19 37.756934 2.180893 -FAST19 1.257291 2.173768 -FAST20 40.273755 2.179442 -FAST20 1.630522 2.170072 -FAST21 54.606548 2.181400 -FAST21 2.321266 2.171643 -FAST22 72.454066 2.178774 -FAST22 5.092888 2.168885 -FAST23 106.753208 2.180347 -FAST23 14.722222 2.170673 -FAST24 171.083201 2.183426 -FAST24 27.575575 2.170623 -COVER 227.219660 2.188654 +RANDOM 0.549307 2.096785 +LEGACY 2.273818 2.058273 +COVER 219.640608 2.188654 +COVER 6.055391 2.188654 +FAST15 67.820700 2.127194 +FAST15 0.824624 2.127194 +FAST16 69.774209 2.145401 +FAST16 0.889737 2.145401 +FAST17 70.027355 2.157544 +FAST17 0.869004 2.157544 +FAST18 68.229652 2.173127 +FAST18 0.930689 2.173127 +FAST19 70.696241 2.179527 +FAST19 1.385515 2.179527 +FAST20 80.618172 2.183233 +FAST20 1.699632 2.183233 +FAST21 96.366254 2.180920 +FAST21 2.606553 2.180920 +FAST22 139.440758 2.184297 +FAST22 5.962606 2.184297 +FAST23 207.791930 2.187666 +FAST23 14.823301 2.187666 +FAST24 322.050385 2.189889 +FAST24 29.294918 2.189889 hg-manifest -NODICT 0.000007 1.866385 -RANDOM 1.086571 2.309485 -LEGACY 9.567507 2.506775 -FAST15 77.811380 2.380461 -FAST15 1.969718 2.317727 -FAST16 75.789019 2.469144 -FAST16 2.051283 2.375815 -FAST17 79.659040 2.539069 -FAST17 1.995394 2.501047 -FAST18 76.281105 2.578095 -FAST18 2.059272 2.564840 -FAST19 79.395382 2.590433 -FAST19 2.354158 2.591024 -FAST20 87.937568 2.597813 -FAST20 2.922189 2.597104 -FAST21 121.760549 2.598408 -FAST21 4.798981 2.600269 -FAST22 155.878461 2.594560 -FAST22 8.151807 2.601047 -FAST23 194.238003 2.596761 -FAST23 15.160578 2.592985 -FAST24 267.425904 2.597657 -FAST24 29.513286 2.600363 -COVER 930.675322 2.582597 +NODICT 0.000008 1.866385 +RANDOM 1.075766 2.309485 +LEGACY 8.688387 2.506775 +COVER 926.024689 2.582597 +COVER 33.630695 2.582597 +FAST15 152.845945 2.377689 +FAST15 2.206285 2.377689 +FAST16 147.772371 2.464814 +FAST16 1.937997 2.464814 +FAST17 147.729498 2.539834 +FAST17 1.966577 2.539834 +FAST18 144.156821 2.576924 +FAST18 1.954106 2.576924 +FAST19 145.678760 2.592479 +FAST19 2.096876 2.592479 +FAST20 159.634674 2.594551 +FAST20 2.568766 2.594551 +FAST21 228.116552 2.597128 +FAST21 4.634508 2.597128 +FAST22 288.890644 2.596971 +FAST22 6.618204 2.596971 +FAST23 377.196211 2.601416 +FAST23 13.497286 2.601416 +FAST24 503.208577 2.602830 +FAST24 29.538585 2.602830 diff --git a/contrib/experimental_dict_builders/benchmarkDictBuilder/benchmark.c b/contrib/experimental_dict_builders/benchmarkDictBuilder/benchmark.c index 9feaae592..a775eae3a 100644 --- a/contrib/experimental_dict_builders/benchmarkDictBuilder/benchmark.c +++ b/contrib/experimental_dict_builders/benchmarkDictBuilder/benchmark.c @@ -277,7 +277,8 @@ int main(int argCount, const char* argv[]) int result = 0; /* Initialize arguments to default values */ - const unsigned k = 200; + unsigned k = 200; + unsigned d = 8; const unsigned cLevel = DEFAULT_CLEVEL; const unsigned dictID = 0; const unsigned maxDictSize = g_defaultMaxDictSize; @@ -360,47 +361,6 @@ int main(int argCount, const char* argv[]) } } - /* for fastCover */ - for (unsigned f = 15; f < 25; f++){ - DISPLAYLEVEL(2, "current f is %u\n", f); - /* for fastCover (optimizing k) */ - { - ZDICT_fastCover_params_t fastParam; - memset(&fastParam, 0, sizeof(fastParam)); - fastParam.zParams = zParams; - fastParam.splitPoint = 1.0; - fastParam.d = 8; - fastParam.f = f; - fastParam.steps = 40; - fastParam.nbThreads = 1; - const int fastOptResult = benchmarkDictBuilder(srcInfo, maxDictSize, NULL, NULL, NULL, &fastParam); - DISPLAYLEVEL(2, "k=%u\nd=%u\nf=%u\nsteps=%u\nsplit=%u\n", fastParam.k, fastParam.d, fastParam.f, fastParam.steps, (unsigned)(fastParam.splitPoint * 100)); - if(fastOptResult) { - result = 1; - goto _cleanup; - } - } - - /* for fastCover (with k provided) */ - { - ZDICT_fastCover_params_t fastParam; - memset(&fastParam, 0, sizeof(fastParam)); - fastParam.zParams = zParams; - fastParam.splitPoint = 1.0; - fastParam.d = 8; - fastParam.f = f; - fastParam.k = 200; - fastParam.steps = 40; - fastParam.nbThreads = 1; - const int fastOptResult = benchmarkDictBuilder(srcInfo, maxDictSize, NULL, NULL, NULL, &fastParam); - DISPLAYLEVEL(2, "k=%u\nd=%u\nf=%u\nsteps=%u\nsplit=%u\n", fastParam.k, fastParam.d, fastParam.f, fastParam.steps, (unsigned)(fastParam.splitPoint * 100)); - if(fastOptResult) { - result = 1; - goto _cleanup; - } - } - } - /* for cover */ { ZDICT_cover_params_t coverParam; @@ -415,8 +375,73 @@ int main(int argCount, const char* argv[]) result = 1; goto _cleanup; } + + k = coverParam.k; + d = coverParam.d; + + /* for COVER with k and d provided */ + ZDICT_cover_params_t covernParam; + memset(&covernParam, 0, sizeof(covernParam)); + covernParam.zParams = zParams; + covernParam.splitPoint = 1.0; + covernParam.steps = 40; + covernParam.nbThreads = 1; + covernParam.k = k; + covernParam.d = d; + const int coverResult = benchmarkDictBuilder(srcInfo, maxDictSize, NULL, &covernParam, NULL, NULL); + DISPLAYLEVEL(2, "k=%u\nd=%u\nsteps=%u\nsplit=%u\n", covernParam.k, covernParam.d, covernParam.steps, (unsigned)(covernParam.splitPoint * 100)); + if(coverResult) { + result = 1; + goto _cleanup; + } } + /* for fastCover */ + for (unsigned f = 15; f < 25; f++){ + DISPLAYLEVEL(2, "current f is %u\n", f); + /* for fastCover (optimizing k and d) */ + { + ZDICT_fastCover_params_t fastParam; + memset(&fastParam, 0, sizeof(fastParam)); + fastParam.zParams = zParams; + fastParam.splitPoint = 1.0; + fastParam.f = f; + fastParam.steps = 40; + fastParam.nbThreads = 1; + const int fastOptResult = benchmarkDictBuilder(srcInfo, maxDictSize, NULL, NULL, NULL, &fastParam); + DISPLAYLEVEL(2, "k=%u\nd=%u\nf=%u\nsteps=%u\nsplit=%u\n", fastParam.k, fastParam.d, fastParam.f, fastParam.steps, (unsigned)(fastParam.splitPoint * 100)); + if(fastOptResult) { + result = 1; + goto _cleanup; + } + + k = fastParam.k; + d = fastParam.d; + } + + + /* for fastCover (with k and d provided) */ + { + ZDICT_fastCover_params_t fastParam; + memset(&fastParam, 0, sizeof(fastParam)); + fastParam.zParams = zParams; + fastParam.splitPoint = 1.0; + fastParam.d = d; + fastParam.f = f; + fastParam.k = k; + fastParam.steps = 40; + fastParam.nbThreads = 1; + const int fastOptResult = benchmarkDictBuilder(srcInfo, maxDictSize, NULL, NULL, NULL, &fastParam); + DISPLAYLEVEL(2, "k=%u\nd=%u\nf=%u\nsteps=%u\nsplit=%u\n", fastParam.k, fastParam.d, fastParam.f, fastParam.steps, (unsigned)(fastParam.splitPoint * 100)); + if(fastOptResult) { + result = 1; + goto _cleanup; + } + } + } + + + /* Free allocated memory */ _cleanup: diff --git a/contrib/experimental_dict_builders/fastCover/fastCover.c b/contrib/experimental_dict_builders/fastCover/fastCover.c index 6f990e0c2..d6b3254ec 100644 --- a/contrib/experimental_dict_builders/fastCover/fastCover.c +++ b/contrib/experimental_dict_builders/fastCover/fastCover.c @@ -267,7 +267,7 @@ static void FASTCOVER_computeFrequency(U32 *freqs, unsigned f, FASTCOVER_ctx_t * size_t currSampleStart = ctx->offsets[i]; size_t currSampleEnd = ctx->offsets[i+1]; start = currSampleStart; - while (start + f < currSampleEnd) { + while (start + ctx->d <= currSampleEnd) { const size_t dmerIndex = FASTCOVER_hashPtrToIndex(ctx->samples + start, f, ctx->d); /* if no dmer with same hash value has been seen in current sample */ if (inCurrSample[dmerIndex] == 0) { From 49b398e93f5357c4311b678a7e4b4d875035f379 Mon Sep 17 00:00:00 2001 From: Jennifer Liu Date: Fri, 27 Jul 2018 13:39:19 -0700 Subject: [PATCH 081/372] Use same param after optimizing cover and fastCover and record k and d for benchmarking --- .../benchmarkDictBuilder/README.md | 211 +++++++++--------- .../benchmarkDictBuilder/benchmark.c | 74 ++---- 2 files changed, 129 insertions(+), 156 deletions(-) diff --git a/contrib/experimental_dict_builders/benchmarkDictBuilder/README.md b/contrib/experimental_dict_builders/benchmarkDictBuilder/README.md index 04866b7e6..654ca4095 100644 --- a/contrib/experimental_dict_builders/benchmarkDictBuilder/README.md +++ b/contrib/experimental_dict_builders/benchmarkDictBuilder/README.md @@ -13,113 +13,114 @@ Benchmark given input files: make ARG= followed by permitted arguments make ARG="in=../../../lib/dictBuilder in=../../../lib/compress" ###Benchmarking Result: -First Cover is optimize cover, second Cover uses optimized d and k from first one. -For every f value of fastCover, the first one is optimize fastCover and the second one uses optimized d and k from first one. +- First Cover is optimize cover, second Cover uses optimized d and k from first one. +- For every f value of fastCover, the first one is optimize fastCover and the second one uses optimized d and k from first one. +- Fourth column is chosen d and fifth column is chosen k github: -NODICT 0.000004 2.999642 -RANDOM 0.146096 8.786957 -LEGACY 0.956888 8.989482 -COVER 56.596152 10.641263 -COVER 4.937047 10.641263 -FAST15 17.722269 10.586461 -FAST15 0.239135 10.586461 -FAST16 18.276179 10.492503 -FAST16 0.265285 10.492503 -FAST17 18.077916 10.611737 -FAST17 0.236573 10.611737 -FAST18 19.510150 10.621586 -FAST18 0.278683 10.621586 -FAST19 18.794350 10.629626 -FAST19 0.307943 10.629626 -FAST20 19.671099 10.610308 -FAST20 0.428814 10.610308 -FAST21 36.527238 10.625733 -FAST21 0.716384 10.625733 -FAST22 83.803521 10.625281 -FAST22 1.290246 10.625281 -FAST23 158.287924 10.602342 -FAST23 3.084848 10.602342 -FAST24 283.630941 10.603379 -FAST24 8.088933 10.603379 +NODICT 0.000004 2.999642 +RANDOM 0.146096 8.786957 +LEGACY 0.956888 8.989482 +COVER 56.596152 10.641263 8 1298 +COVER 4.937047 10.641263 8 1298 +FAST15 17.722269 10.586461 8 1778 +FAST15 0.239135 10.586461 8 1778 +FAST16 18.276179 10.492503 6 1778 +FAST16 0.265285 10.492503 6 1778 +FAST17 18.077916 10.611737 8 1778 +FAST17 0.236573 10.611737 8 1778 +FAST18 19.510150 10.621586 8 1778 +FAST18 0.278683 10.621586 8 1778 +FAST19 18.794350 10.629626 8 1778 +FAST19 0.307943 10.629626 8 1778 +FAST20 19.671099 10.610308 8 1778 +FAST20 0.428814 10.610308 8 1778 +FAST21 36.527238 10.625733 8 1778 +FAST21 0.716384 10.625733 8 1778 +FAST22 83.803521 10.625281 8 1778 +FAST22 1.290246 10.625281 8 1778 +FAST23 158.287924 10.602342 8 1778 +FAST23 3.084848 10.602342 8 1778 +FAST24 283.630941 10.603379 8 1778 +FAST24 8.088933 10.603379 8 1778 -hg-commands -NODICT 0.000007 2.425291 -RANDOM 0.084010 3.489515 -LEGACY 0.926763 3.911896 -COVER 62.036915 4.131136 -COVER 2.194398 4.131136 -FAST15 12.169025 3.903719 -FAST15 0.156552 3.903719 -FAST16 11.886255 4.005077 -FAST16 0.155506 4.005077 -FAST17 11.886955 4.097811 -FAST17 0.176327 4.097811 -FAST18 12.544698 4.136081 -FAST18 0.171796 4.136081 -FAST19 12.920868 4.166021 -FAST19 0.207029 4.166021 -FAST20 15.771429 4.163740 -FAST20 0.258685 4.163740 -FAST21 33.165829 4.157057 -FAST21 0.663088 4.157057 -FAST22 68.779201 4.158195 -FAST22 1.568439 4.158195 -FAST23 121.921931 4.161450 -FAST23 2.498972 4.161450 -FAST24 221.990451 4.159658 -FAST24 5.793594 4.159658 +hg-commands: +NODICT 0.000007 2.425291 +RANDOM 0.084010 3.489515 +LEGACY 0.926763 3.911896 +COVER 62.036915 4.131136 8 386 +COVER 2.194398 4.131136 8 386 +FAST15 12.169025 3.903719 6 1106 +FAST15 0.156552 3.903719 6 1106 +FAST16 11.886255 4.005077 8 530 +FAST16 0.155506 4.005077 8 530 +FAST17 11.886955 4.097811 8 818 +FAST17 0.176327 4.097811 8 818 +FAST18 12.544698 4.136081 8 770 +FAST18 0.171796 4.136081 8 770 +FAST19 12.920868 4.166021 8 530 +FAST19 0.207029 4.166021 8 530 +FAST20 15.771429 4.163740 8 482 +FAST20 0.258685 4.163740 8 482 +FAST21 33.165829 4.157057 8 434 +FAST21 0.663088 4.157057 8 434 +FAST22 68.779201 4.158195 8 290 +FAST22 1.568439 4.158195 8 290 +FAST23 121.921931 4.161450 8 434 +FAST23 2.498972 4.161450 8 434 +FAST24 221.990451 4.159658 8 338 +FAST24 5.793594 4.159658 8 338 -hg-changelog -NODICT 0.000004 1.377613 -RANDOM 0.549307 2.096785 -LEGACY 2.273818 2.058273 -COVER 219.640608 2.188654 -COVER 6.055391 2.188654 -FAST15 67.820700 2.127194 -FAST15 0.824624 2.127194 -FAST16 69.774209 2.145401 -FAST16 0.889737 2.145401 -FAST17 70.027355 2.157544 -FAST17 0.869004 2.157544 -FAST18 68.229652 2.173127 -FAST18 0.930689 2.173127 -FAST19 70.696241 2.179527 -FAST19 1.385515 2.179527 -FAST20 80.618172 2.183233 -FAST20 1.699632 2.183233 -FAST21 96.366254 2.180920 -FAST21 2.606553 2.180920 -FAST22 139.440758 2.184297 -FAST22 5.962606 2.184297 -FAST23 207.791930 2.187666 -FAST23 14.823301 2.187666 -FAST24 322.050385 2.189889 -FAST24 29.294918 2.189889 +hg-changelog: +NODICT 0.000004 1.377613 +RANDOM 0.549307 2.096785 +LEGACY 2.273818 2.058273 +COVER 219.640608 2.188654 8 98 +COVER 6.055391 2.188654 8 98 +FAST15 67.820700 2.127194 8 866 +FAST15 0.824624 2.127194 8 866 +FAST16 69.774209 2.145401 8 338 +FAST16 0.889737 2.145401 8 338 +FAST17 70.027355 2.157544 8 194 +FAST17 0.869004 2.157544 8 194 +FAST18 68.229652 2.173127 8 98 +FAST18 0.930689 2.173127 8 98 +FAST19 70.696241 2.179527 8 98 +FAST19 1.385515 2.179527 8 98 +FAST20 80.618172 2.183233 6 98 +FAST20 1.699632 2.183233 6 98 +FAST21 96.366254 2.180920 8 98 +FAST21 2.606553 2.180920 8 98 +FAST22 139.440758 2.184297 8 98 +FAST22 5.962606 2.184297 8 98 +FAST23 207.791930 2.187666 6 98 +FAST23 14.823301 2.187666 6 98 +FAST24 322.050385 2.189889 6 98 +FAST24 29.294918 2.189889 6 98 -hg-manifest -NODICT 0.000008 1.866385 -RANDOM 1.075766 2.309485 -LEGACY 8.688387 2.506775 -COVER 926.024689 2.582597 -COVER 33.630695 2.582597 -FAST15 152.845945 2.377689 -FAST15 2.206285 2.377689 -FAST16 147.772371 2.464814 -FAST16 1.937997 2.464814 -FAST17 147.729498 2.539834 -FAST17 1.966577 2.539834 -FAST18 144.156821 2.576924 -FAST18 1.954106 2.576924 -FAST19 145.678760 2.592479 -FAST19 2.096876 2.592479 -FAST20 159.634674 2.594551 -FAST20 2.568766 2.594551 -FAST21 228.116552 2.597128 -FAST21 4.634508 2.597128 -FAST22 288.890644 2.596971 -FAST22 6.618204 2.596971 -FAST23 377.196211 2.601416 -FAST23 13.497286 2.601416 -FAST24 503.208577 2.602830 -FAST24 29.538585 2.602830 +hg-manifest: +NODICT 0.000008 1.866385 +RANDOM 1.075766 2.309485 +LEGACY 8.688387 2.506775 +COVER 926.024689 2.582597 8 434 +COVER 33.630695 2.582597 8 434 +FAST15 152.845945 2.377689 8 1682 +FAST15 2.206285 2.377689 8 1682 +FAST16 147.772371 2.464814 8 1538 +FAST16 1.937997 2.464814 8 1538 +FAST17 147.729498 2.539834 6 1826 +FAST17 1.966577 2.539834 6 1826 +FAST18 144.156821 2.576924 8 1922 +FAST18 1.954106 2.576924 8 1922 +FAST19 145.678760 2.592479 6 290 +FAST19 2.096876 2.592479 6 290 +FAST20 159.634674 2.594551 8 194 +FAST20 2.568766 2.594551 8 194 +FAST21 228.116552 2.597128 6 194 +FAST21 4.634508 2.597128 6 194 +FAST22 288.890644 2.596971 6 386 +FAST22 6.618204 2.596971 6 386 +FAST23 377.196211 2.601416 8 194 +FAST23 13.497286 2.601416 8 194 +FAST24 503.208577 2.602830 6 194 +FAST24 29.538585 2.602830 6 194 diff --git a/contrib/experimental_dict_builders/benchmarkDictBuilder/benchmark.c b/contrib/experimental_dict_builders/benchmarkDictBuilder/benchmark.c index a775eae3a..75008a087 100644 --- a/contrib/experimental_dict_builders/benchmarkDictBuilder/benchmark.c +++ b/contrib/experimental_dict_builders/benchmarkDictBuilder/benchmark.c @@ -251,7 +251,7 @@ int benchmarkDictBuilder(sampleInfo *srcInfo, unsigned maxDictSize, ZDICT_random result = 1; goto _cleanup; } - DISPLAYLEVEL(2, "%s took %f seconds to execute \n", name, timeSec); + DISPLAYLEVEL(1, "%s took %f seconds to execute \n", name, timeSec); /* Calculate compression ratio */ const double cRatio = compressWithDict(srcInfo, dInfo, cLevel, displayLevel); @@ -261,7 +261,7 @@ int benchmarkDictBuilder(sampleInfo *srcInfo, unsigned maxDictSize, ZDICT_random goto _cleanup; } - DISPLAYLEVEL(2, "Compression ratio with %s dictionary is %f\n", name, cRatio); + DISPLAYLEVEL(1, "Compression ratio with %s dictionary is %f\n", name, cRatio); _cleanup: freeDictInfo(dInfo); @@ -376,73 +376,45 @@ int main(int argCount, const char* argv[]) goto _cleanup; } - k = coverParam.k; - d = coverParam.d; - - /* for COVER with k and d provided */ - ZDICT_cover_params_t covernParam; - memset(&covernParam, 0, sizeof(covernParam)); - covernParam.zParams = zParams; - covernParam.splitPoint = 1.0; - covernParam.steps = 40; - covernParam.nbThreads = 1; - covernParam.k = k; - covernParam.d = d; - const int coverResult = benchmarkDictBuilder(srcInfo, maxDictSize, NULL, &covernParam, NULL, NULL); - DISPLAYLEVEL(2, "k=%u\nd=%u\nsteps=%u\nsplit=%u\n", covernParam.k, covernParam.d, covernParam.steps, (unsigned)(covernParam.splitPoint * 100)); + const int coverResult = benchmarkDictBuilder(srcInfo, maxDictSize, NULL, &coverParam, NULL, NULL); + DISPLAYLEVEL(2, "k=%u\nd=%u\nsteps=%u\nsplit=%u\n", coverParam.k, coverParam.d, coverParam.steps, (unsigned)(coverParam.splitPoint * 100)); if(coverResult) { result = 1; goto _cleanup; } + } /* for fastCover */ for (unsigned f = 15; f < 25; f++){ DISPLAYLEVEL(2, "current f is %u\n", f); /* for fastCover (optimizing k and d) */ - { - ZDICT_fastCover_params_t fastParam; - memset(&fastParam, 0, sizeof(fastParam)); - fastParam.zParams = zParams; - fastParam.splitPoint = 1.0; - fastParam.f = f; - fastParam.steps = 40; - fastParam.nbThreads = 1; - const int fastOptResult = benchmarkDictBuilder(srcInfo, maxDictSize, NULL, NULL, NULL, &fastParam); - DISPLAYLEVEL(2, "k=%u\nd=%u\nf=%u\nsteps=%u\nsplit=%u\n", fastParam.k, fastParam.d, fastParam.f, fastParam.steps, (unsigned)(fastParam.splitPoint * 100)); - if(fastOptResult) { - result = 1; - goto _cleanup; - } - - k = fastParam.k; - d = fastParam.d; + ZDICT_fastCover_params_t fastParam; + memset(&fastParam, 0, sizeof(fastParam)); + fastParam.zParams = zParams; + fastParam.splitPoint = 1.0; + fastParam.f = f; + fastParam.steps = 40; + fastParam.nbThreads = 1; + const int fastOptResult = benchmarkDictBuilder(srcInfo, maxDictSize, NULL, NULL, NULL, &fastParam); + DISPLAYLEVEL(2, "k=%u\nd=%u\nf=%u\nsteps=%u\nsplit=%u\n", fastParam.k, fastParam.d, fastParam.f, fastParam.steps, (unsigned)(fastParam.splitPoint * 100)); + if(fastOptResult) { + result = 1; + goto _cleanup; } /* for fastCover (with k and d provided) */ - { - ZDICT_fastCover_params_t fastParam; - memset(&fastParam, 0, sizeof(fastParam)); - fastParam.zParams = zParams; - fastParam.splitPoint = 1.0; - fastParam.d = d; - fastParam.f = f; - fastParam.k = k; - fastParam.steps = 40; - fastParam.nbThreads = 1; - const int fastOptResult = benchmarkDictBuilder(srcInfo, maxDictSize, NULL, NULL, NULL, &fastParam); - DISPLAYLEVEL(2, "k=%u\nd=%u\nf=%u\nsteps=%u\nsplit=%u\n", fastParam.k, fastParam.d, fastParam.f, fastParam.steps, (unsigned)(fastParam.splitPoint * 100)); - if(fastOptResult) { - result = 1; - goto _cleanup; - } + const int fastResult = benchmarkDictBuilder(srcInfo, maxDictSize, NULL, NULL, NULL, &fastParam); + DISPLAYLEVEL(2, "k=%u\nd=%u\nf=%u\nsteps=%u\nsplit=%u\n", fastParam.k, fastParam.d, fastParam.f, fastParam.steps, (unsigned)(fastParam.splitPoint * 100)); + if(fastResult) { + result = 1; + goto _cleanup; } + } - - /* Free allocated memory */ _cleanup: UTIL_freeFileList(extendedFileList, fileNamesBuf); From 61262f6c0dc137e078bbc4cd1131fc3b88657414 Mon Sep 17 00:00:00 2001 From: Jennifer Liu Date: Fri, 27 Jul 2018 16:51:38 -0700 Subject: [PATCH 082/372] Save segmentFreqs in ctx instead of malloc and memset in SelectSegment --- .../benchmarkDictBuilder/README.md | 113 ++++++++++++++++++ .../benchmarkDictBuilder/test.sh | 10 +- .../fastCover/Makefile | 6 +- .../fastCover/fastCover.c | 28 +++-- 4 files changed, 141 insertions(+), 16 deletions(-) diff --git a/contrib/experimental_dict_builders/benchmarkDictBuilder/README.md b/contrib/experimental_dict_builders/benchmarkDictBuilder/README.md index 654ca4095..a818e6eb4 100644 --- a/contrib/experimental_dict_builders/benchmarkDictBuilder/README.md +++ b/contrib/experimental_dict_builders/benchmarkDictBuilder/README.md @@ -17,6 +17,8 @@ make ARG="in=../../../lib/dictBuilder in=../../../lib/compress" - For every f value of fastCover, the first one is optimize fastCover and the second one uses optimized d and k from first one. - Fourth column is chosen d and fifth column is chosen k +Version 1: + github: NODICT 0.000004 2.999642 RANDOM 0.146096 8.786957 @@ -124,3 +126,114 @@ FAST23 377.196211 2.601416 8 194 FAST23 13.497286 2.601416 8 194 FAST24 503.208577 2.602830 6 194 FAST24 29.538585 2.602830 6 194 + +--------------------------------------------------------------- +Version 2 (save segmentFreqs in ctx instead of malloc and memset in every call to SelectSegment): + +github: +NODICT 0.000005 2.999642 +RANDOM 0.141553 8.786957 +LEGACY 0.904340 8.989482 +COVER 53.621302 10.641263 8 1298 +COVER 4.085037 10.641263 8 1298 +FAST15 17.636211 10.586461 8 1778 +FAST15 0.221236 10.586461 8 1778 +FAST16 18.716259 10.492503 6 1778 +FAST16 0.251522 10.492503 6 1778 +FAST17 17.614391 10.611737 8 1778 +FAST17 0.241011 10.611737 8 1778 +FAST18 19.926270 10.621586 8 1778 +FAST18 0.287195 10.621586 8 1778 +FAST19 19.626808 10.629626 8 1778 +FAST19 0.340191 10.629626 8 1778 +FAST20 18.918657 10.610308 8 1778 +FAST20 0.463307 10.610308 8 1778 +FAST21 20.502362 10.625733 8 1778 +FAST21 0.638202 10.625733 8 1778 +FAST22 22.702695 10.625281 8 1778 +FAST22 1.353399 10.625281 8 1778 +FAST23 28.041990 10.602342 8 1778 +FAST23 3.029502 10.602342 8 1778 +FAST24 35.662961 10.603379 8 1778 +FAST24 6.524258 10.603379 8 1778 + +hg-commands: +NODICT 0.000005 2.425291 +RANDOM 0.080469 3.489515 +LEGACY 0.794417 3.911896 +COVER 54.198788 4.131136 8 386 +COVER 2.191729 4.131136 8 386 +FAST15 11.852793 3.903719 6 1106 +FAST15 0.175406 3.903719 6 1106 +FAST16 12.863315 4.005077 8 530 +FAST16 0.158410 4.005077 8 530 +FAST17 11.977917 4.097811 8 818 +FAST17 0.162381 4.097811 8 818 +FAST18 11.749304 4.136081 8 770 +FAST18 0.173242 4.136081 8 770 +FAST19 11.905785 4.166021 8 530 +FAST19 0.186403 4.166021 8 530 +FAST20 13.293999 4.163740 8 482 +FAST20 0.241508 4.163740 8 482 +FAST21 16.623177 4.157057 8 434 +FAST21 0.372647 4.157057 8 434 +FAST22 20.918409 4.158195 8 290 +FAST22 0.570431 4.158195 8 290 +FAST23 21.762805 4.161450 8 434 +FAST23 1.162206 4.161450 8 434 +FAST24 29.133745 4.159658 8 338 +FAST24 3.054376 4.159658 8 338 + +hg-changelog: +NODICT 0.000006 1.377613 +RANDOM 0.601346 2.096785 +LEGACY 2.544973 2.058273 +COVER 222.639708 2.188654 8 98 +COVER 6.072892 2.188654 8 98 +FAST15 70.394523 2.127194 8 866 +FAST15 0.899766 2.127194 8 866 +FAST16 69.845529 2.145401 8 338 +FAST16 0.881569 2.145401 8 338 +FAST17 69.382431 2.157544 8 194 +FAST17 0.943291 2.157544 8 194 +FAST18 71.348283 2.173127 8 98 +FAST18 1.034765 2.173127 8 98 +FAST19 71.380923 2.179527 8 98 +FAST19 1.254700 2.179527 8 98 +FAST20 72.802714 2.183233 6 98 +FAST20 1.368704 2.183233 6 98 +FAST21 82.042339 2.180920 8 98 +FAST21 2.213864 2.180920 8 98 +FAST22 90.666200 2.184297 8 98 +FAST22 3.590399 2.184297 8 98 +FAST23 108.926377 2.187666 6 98 +FAST23 8.723759 2.187666 6 98 +FAST24 134.296232 2.189889 6 98 +FAST24 19.396532 2.189889 6 98 + +hg-manifest: +NODICT 0.000005 1.866385 +RANDOM 0.982192 2.309485 +LEGACY 9.507729 2.506775 +COVER 922.742066 2.582597 8 434 +COVER 36.500276 2.582597 8 434 +FAST15 163.886717 2.377689 8 1682 +FAST15 2.107328 2.377689 8 1682 +FAST16 152.684592 2.464814 8 1538 +FAST16 2.157789 2.464814 8 1538 +FAST17 154.463459 2.539834 6 1826 +FAST17 2.282455 2.539834 6 1826 +FAST18 155.540044 2.576924 8 1922 +FAST18 2.101807 2.576924 8 1922 +FAST19 152.650343 2.592479 6 290 +FAST19 2.359461 2.592479 6 290 +FAST20 174.623634 2.594551 8 194 +FAST20 2.870022 2.594551 8 194 +FAST21 219.876653 2.597128 6 194 +FAST21 4.386269 2.597128 6 194 +FAST22 247.986803 2.596971 6 386 +FAST22 6.201144 2.596971 6 386 +FAST23 276.051806 2.601416 8 194 +FAST23 11.613477 2.601416 8 194 +FAST24 328.234024 2.602830 6 194 +FAST24 26.710364 2.602830 6 194 diff --git a/contrib/experimental_dict_builders/benchmarkDictBuilder/test.sh b/contrib/experimental_dict_builders/benchmarkDictBuilder/test.sh index 5eaf5930a..e5508ded3 100644 --- a/contrib/experimental_dict_builders/benchmarkDictBuilder/test.sh +++ b/contrib/experimental_dict_builders/benchmarkDictBuilder/test.sh @@ -1,2 +1,8 @@ -echo "Benchmark with in=../../lib/common" -./benchmark in=../../../lib/common +echo "-----------------github--------------------" +./benchmark in=github +echo "-----------------hg-commands--------------------" +./benchmark in=hg-commands +echo "-----------------hg-changelog--------------------" +./benchmark in=hg-changelog +echo "------------------hg-manifest-------------------" +./benchmark in=hg-manifest diff --git a/contrib/experimental_dict_builders/fastCover/Makefile b/contrib/experimental_dict_builders/fastCover/Makefile index 9c56013d8..4a7cc17d0 100644 --- a/contrib/experimental_dict_builders/fastCover/Makefile +++ b/contrib/experimental_dict_builders/fastCover/Makefile @@ -1,7 +1,7 @@ ARG := CC ?= gcc -CFLAGS ?= -O3 +CFLAGS ?= -O3 -g INCLUDES := -I ../../../programs -I ../randomDictBuilder -I ../../../lib/common -I ../../../lib -I ../../../lib/dictBuilder IO_FILE := ../randomDictBuilder/io.c @@ -9,7 +9,7 @@ IO_FILE := ../randomDictBuilder/io.c TEST_INPUT := ../../../lib TEST_OUTPUT := fastCoverDict -all: main run clean +all: main run .PHONY: test test: main testrun testshell clean @@ -32,7 +32,7 @@ io.o: $(IO_FILE) $(CC) $(CFLAGS) $(INCLUDES) -c $(IO_FILE) libzstd.a: - $(MAKE) -C ../../../lib libzstd.a + $(MAKE) MOREFLAGS=-g -C ../../../lib libzstd.a mv ../../../lib/libzstd.a . .PHONY: testrun diff --git a/contrib/experimental_dict_builders/fastCover/fastCover.c b/contrib/experimental_dict_builders/fastCover/fastCover.c index d6b3254ec..3c1aa951c 100644 --- a/contrib/experimental_dict_builders/fastCover/fastCover.c +++ b/contrib/experimental_dict_builders/fastCover/fastCover.c @@ -82,6 +82,7 @@ typedef struct { size_t nbTestSamples; size_t nbDmers; U32 *freqs; + U16 *segmentFreqs; unsigned d; } FASTCOVER_ctx_t; @@ -142,9 +143,6 @@ static FASTCOVER_segment_t FASTCOVER_selectSegment(const FASTCOVER_ctx_t *ctx, activeSegment.end = begin; activeSegment.score = 0; { - /* Keep track of number of times an index has been seen in current segment */ - U16* currfreqs =(U16 *)malloc((1 << parameters.f) * sizeof(U16)); - memset(currfreqs, 0, (1 << parameters.f) * sizeof(*currfreqs)); /* Slide the activeSegment through the whole epoch. * Save the best segment in bestSegment. */ @@ -152,19 +150,19 @@ static FASTCOVER_segment_t FASTCOVER_selectSegment(const FASTCOVER_ctx_t *ctx, /* Get hash value of current dmer */ const size_t index = FASTCOVER_hashPtrToIndex(ctx->samples + activeSegment.end, parameters.f, ctx->d); /* Add frequency of this index to score if this is the first occurence of index in active segment */ - if (currfreqs[index] == 0) { + if (ctx->segmentFreqs[index] == 0) { activeSegment.score += freqs[index]; } - currfreqs[index] += 1; + ctx->segmentFreqs[index] += 1; /* Increment end of segment */ activeSegment.end += 1; /* If the window is now too large, drop the first position */ if (activeSegment.end - activeSegment.begin == dmersInK + 1) { /* Get hash value of the dmer to be eliminated from active segment */ const size_t delIndex = FASTCOVER_hashPtrToIndex(ctx->samples + activeSegment.begin, parameters.f, ctx->d); - currfreqs[delIndex] -= 1; + ctx->segmentFreqs[delIndex] -= 1; /* Subtract frequency of this index from score if this is the last occurrence of this index in active segment */ - if (currfreqs[delIndex] == 0) { + if (ctx->segmentFreqs[delIndex] == 0) { activeSegment.score -= freqs[delIndex]; } /* Increment start of segment */ @@ -175,7 +173,12 @@ static FASTCOVER_segment_t FASTCOVER_selectSegment(const FASTCOVER_ctx_t *ctx, bestSegment = activeSegment; } } - free(currfreqs); + /* Zero out rest of segmentFreqs array */ + while (activeSegment.begin < end) { + const size_t delIndex = FASTCOVER_hashPtrToIndex(ctx->samples + activeSegment.begin, parameters.f, ctx->d); + ctx->segmentFreqs[delIndex] -= 1; + activeSegment.begin += 1; + } } { /* Trim off the zero frequency head and tail from the segment. */ @@ -245,6 +248,10 @@ static void FASTCOVER_ctx_destroy(FASTCOVER_ctx_t *ctx) { if (!ctx) { return; } + if (ctx->segmentFreqs) { + free(ctx->segmentFreqs); + ctx->segmentFreqs = NULL; + } if (ctx->freqs) { free(ctx->freqs); ctx->freqs = NULL; @@ -347,9 +354,8 @@ static int FASTCOVER_ctx_init(FASTCOVER_ctx_t *ctx, const void *samplesBuffer, } /* Initialize frequency array of size 2^f */ - ctx->freqs =(U32 *)malloc((1 << f) * sizeof(U32)); - memset(ctx->freqs, 0, (1 << f) * sizeof(U32)); - + ctx->freqs = (U32 *)calloc((1 << f), sizeof(U32)); + ctx->segmentFreqs = (U16 *)calloc((1 << f), sizeof(U16)); DISPLAYLEVEL(2, "Computing frequencies\n"); FASTCOVER_computeFrequency(ctx->freqs, f, ctx); From 96d84ee235f4d6cbf71c415a1a0327235751ba86 Mon Sep 17 00:00:00 2001 From: Jennifer Liu Date: Fri, 27 Jul 2018 16:54:05 -0700 Subject: [PATCH 083/372] Revert test.sh --- .../benchmarkDictBuilder/test.sh | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/contrib/experimental_dict_builders/benchmarkDictBuilder/test.sh b/contrib/experimental_dict_builders/benchmarkDictBuilder/test.sh index e5508ded3..5eaf5930a 100644 --- a/contrib/experimental_dict_builders/benchmarkDictBuilder/test.sh +++ b/contrib/experimental_dict_builders/benchmarkDictBuilder/test.sh @@ -1,8 +1,2 @@ -echo "-----------------github--------------------" -./benchmark in=github -echo "-----------------hg-commands--------------------" -./benchmark in=hg-commands -echo "-----------------hg-changelog--------------------" -./benchmark in=hg-changelog -echo "------------------hg-manifest-------------------" -./benchmark in=hg-manifest +echo "Benchmark with in=../../lib/common" +./benchmark in=../../../lib/common From 53ef22a4bc3844f860531dce31481db8b6fcd9bf Mon Sep 17 00:00:00 2001 From: Jennifer Liu Date: Fri, 27 Jul 2018 16:56:50 -0700 Subject: [PATCH 084/372] Undo deleting clean in make --- contrib/experimental_dict_builders/fastCover/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/experimental_dict_builders/fastCover/Makefile b/contrib/experimental_dict_builders/fastCover/Makefile index 4a7cc17d0..3ba24790c 100644 --- a/contrib/experimental_dict_builders/fastCover/Makefile +++ b/contrib/experimental_dict_builders/fastCover/Makefile @@ -9,7 +9,7 @@ IO_FILE := ../randomDictBuilder/io.c TEST_INPUT := ../../../lib TEST_OUTPUT := fastCoverDict -all: main run +all: main run clean .PHONY: test test: main testrun testshell clean From 9889bca530a2b52615eb1cd06260e9cd0c29e001 Mon Sep 17 00:00:00 2001 From: Nick Terrell Date: Fri, 27 Jul 2018 17:30:03 -0700 Subject: [PATCH 085/372] [FSE] Fix division by zero When the primary normalization method fails, and `(1 << tableLog) == (maxSymbolValue + 1)`, and every symbol gets assigned normalized weight 1 or -1 in the first loop, then the next division can raise `SIGFPE`. --- lib/compress/fse_compress.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/compress/fse_compress.c b/lib/compress/fse_compress.c index 07b3ab89b..95e7c1c7e 100644 --- a/lib/compress/fse_compress.c +++ b/lib/compress/fse_compress.c @@ -394,6 +394,9 @@ static size_t FSE_normalizeM2(short* norm, U32 tableLog, const unsigned* count, } ToDistribute = (1 << tableLog) - distributed; + if (ToDistribute == 0) + return 0; + if ((total / ToDistribute) > lowOne) { /* risk of rounding to zero */ lowOne = (U32)((total * 3) / (ToDistribute * 2)); From 51b109c1b5991d3a9bac7bbd5e82065a816777cb Mon Sep 17 00:00:00 2001 From: Jennifer Liu Date: Fri, 27 Jul 2018 17:31:33 -0700 Subject: [PATCH 086/372] Delete old benchmarking result --- .../benchmarkDictBuilder/README.md | 113 ------------------ 1 file changed, 113 deletions(-) diff --git a/contrib/experimental_dict_builders/benchmarkDictBuilder/README.md b/contrib/experimental_dict_builders/benchmarkDictBuilder/README.md index a818e6eb4..20fbde954 100644 --- a/contrib/experimental_dict_builders/benchmarkDictBuilder/README.md +++ b/contrib/experimental_dict_builders/benchmarkDictBuilder/README.md @@ -17,119 +17,6 @@ make ARG="in=../../../lib/dictBuilder in=../../../lib/compress" - For every f value of fastCover, the first one is optimize fastCover and the second one uses optimized d and k from first one. - Fourth column is chosen d and fifth column is chosen k -Version 1: - -github: -NODICT 0.000004 2.999642 -RANDOM 0.146096 8.786957 -LEGACY 0.956888 8.989482 -COVER 56.596152 10.641263 8 1298 -COVER 4.937047 10.641263 8 1298 -FAST15 17.722269 10.586461 8 1778 -FAST15 0.239135 10.586461 8 1778 -FAST16 18.276179 10.492503 6 1778 -FAST16 0.265285 10.492503 6 1778 -FAST17 18.077916 10.611737 8 1778 -FAST17 0.236573 10.611737 8 1778 -FAST18 19.510150 10.621586 8 1778 -FAST18 0.278683 10.621586 8 1778 -FAST19 18.794350 10.629626 8 1778 -FAST19 0.307943 10.629626 8 1778 -FAST20 19.671099 10.610308 8 1778 -FAST20 0.428814 10.610308 8 1778 -FAST21 36.527238 10.625733 8 1778 -FAST21 0.716384 10.625733 8 1778 -FAST22 83.803521 10.625281 8 1778 -FAST22 1.290246 10.625281 8 1778 -FAST23 158.287924 10.602342 8 1778 -FAST23 3.084848 10.602342 8 1778 -FAST24 283.630941 10.603379 8 1778 -FAST24 8.088933 10.603379 8 1778 - -hg-commands: -NODICT 0.000007 2.425291 -RANDOM 0.084010 3.489515 -LEGACY 0.926763 3.911896 -COVER 62.036915 4.131136 8 386 -COVER 2.194398 4.131136 8 386 -FAST15 12.169025 3.903719 6 1106 -FAST15 0.156552 3.903719 6 1106 -FAST16 11.886255 4.005077 8 530 -FAST16 0.155506 4.005077 8 530 -FAST17 11.886955 4.097811 8 818 -FAST17 0.176327 4.097811 8 818 -FAST18 12.544698 4.136081 8 770 -FAST18 0.171796 4.136081 8 770 -FAST19 12.920868 4.166021 8 530 -FAST19 0.207029 4.166021 8 530 -FAST20 15.771429 4.163740 8 482 -FAST20 0.258685 4.163740 8 482 -FAST21 33.165829 4.157057 8 434 -FAST21 0.663088 4.157057 8 434 -FAST22 68.779201 4.158195 8 290 -FAST22 1.568439 4.158195 8 290 -FAST23 121.921931 4.161450 8 434 -FAST23 2.498972 4.161450 8 434 -FAST24 221.990451 4.159658 8 338 -FAST24 5.793594 4.159658 8 338 - -hg-changelog: -NODICT 0.000004 1.377613 -RANDOM 0.549307 2.096785 -LEGACY 2.273818 2.058273 -COVER 219.640608 2.188654 8 98 -COVER 6.055391 2.188654 8 98 -FAST15 67.820700 2.127194 8 866 -FAST15 0.824624 2.127194 8 866 -FAST16 69.774209 2.145401 8 338 -FAST16 0.889737 2.145401 8 338 -FAST17 70.027355 2.157544 8 194 -FAST17 0.869004 2.157544 8 194 -FAST18 68.229652 2.173127 8 98 -FAST18 0.930689 2.173127 8 98 -FAST19 70.696241 2.179527 8 98 -FAST19 1.385515 2.179527 8 98 -FAST20 80.618172 2.183233 6 98 -FAST20 1.699632 2.183233 6 98 -FAST21 96.366254 2.180920 8 98 -FAST21 2.606553 2.180920 8 98 -FAST22 139.440758 2.184297 8 98 -FAST22 5.962606 2.184297 8 98 -FAST23 207.791930 2.187666 6 98 -FAST23 14.823301 2.187666 6 98 -FAST24 322.050385 2.189889 6 98 -FAST24 29.294918 2.189889 6 98 - -hg-manifest: -NODICT 0.000008 1.866385 -RANDOM 1.075766 2.309485 -LEGACY 8.688387 2.506775 -COVER 926.024689 2.582597 8 434 -COVER 33.630695 2.582597 8 434 -FAST15 152.845945 2.377689 8 1682 -FAST15 2.206285 2.377689 8 1682 -FAST16 147.772371 2.464814 8 1538 -FAST16 1.937997 2.464814 8 1538 -FAST17 147.729498 2.539834 6 1826 -FAST17 1.966577 2.539834 6 1826 -FAST18 144.156821 2.576924 8 1922 -FAST18 1.954106 2.576924 8 1922 -FAST19 145.678760 2.592479 6 290 -FAST19 2.096876 2.592479 6 290 -FAST20 159.634674 2.594551 8 194 -FAST20 2.568766 2.594551 8 194 -FAST21 228.116552 2.597128 6 194 -FAST21 4.634508 2.597128 6 194 -FAST22 288.890644 2.596971 6 386 -FAST22 6.618204 2.596971 6 386 -FAST23 377.196211 2.601416 8 194 -FAST23 13.497286 2.601416 8 194 -FAST24 503.208577 2.602830 6 194 -FAST24 29.538585 2.602830 6 194 - ---------------------------------------------------------------- -Version 2 (save segmentFreqs in ctx instead of malloc and memset in every call to SelectSegment): - github: NODICT 0.000005 2.999642 RANDOM 0.141553 8.786957 From 3d4b09a5aa7592b86df77a7956ea51b70799d135 Mon Sep 17 00:00:00 2001 From: cyan4973 Date: Mon, 30 Jul 2018 16:29:20 +0200 Subject: [PATCH 087/372] support %zu under mingw --- tests/Makefile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/Makefile b/tests/Makefile index 813380cc2..0955272ef 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -27,6 +27,9 @@ DEBUGLEVEL ?= 1 DEBUGFLAGS = -g -DDEBUGLEVEL=$(DEBUGLEVEL) CPPFLAGS += -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(ZSTDDIR)/compress \ -I$(ZSTDDIR)/dictBuilder -I$(ZSTDDIR)/deprecated -I$(PRGDIR) +ifeq ($(OS),Windows_NT) # MinGW assumed +CPPFLAGS += -D__USE_MINGW_ANSI_STDIO +endif CFLAGS ?= -O3 CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \ -Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \ From c738a2c795380c2a75810c48f320df93fd8bd2ea Mon Sep 17 00:00:00 2001 From: cyan4973 Date: Mon, 30 Jul 2018 16:44:20 +0200 Subject: [PATCH 088/372] ensure appveyor test fails due to formatting error to catch %zu incompatibility --- appveyor.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/appveyor.yml b/appveyor.yml index 742f61206..42eb71d64 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -181,7 +181,7 @@ - COMPILER: "gcc" HOST: "mingw" PLATFORM: "x64" - SCRIPT: "make allzstd" + SCRIPT: "CPPFLAGS=-DDEBUGLEVEL=2 CFLAGS=-Werror make -j allzstd" - COMPILER: "gcc" HOST: "mingw" PLATFORM: "x86" From 3f535007e40f1e43ea1513c2ff9356114f1be024 Mon Sep 17 00:00:00 2001 From: cyan4973 Date: Mon, 30 Jul 2018 16:56:18 +0200 Subject: [PATCH 089/372] fix %zu support under minGW and relevant test on Appveyor --- appveyor.yml | 2 +- lib/Makefile | 7 +++++-- programs/Makefile | 5 ++++- tests/Makefile | 2 +- 4 files changed, 11 insertions(+), 5 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index 42eb71d64..c9f88dce6 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -181,7 +181,7 @@ - COMPILER: "gcc" HOST: "mingw" PLATFORM: "x64" - SCRIPT: "CPPFLAGS=-DDEBUGLEVEL=2 CFLAGS=-Werror make -j allzstd" + SCRIPT: "CPPFLAGS=-DDEBUGLEVEL=2 CFLAGS=-Werror make -j allzstd DEBUGLEVEL=2" - COMPILER: "gcc" HOST: "mingw" PLATFORM: "x86" diff --git a/lib/Makefile b/lib/Makefile index 9cedd53b7..01689c6d5 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -19,6 +19,9 @@ LIBVER := $(shell echo $(LIBVER_SCRIPT)) VERSION?= $(LIBVER) CPPFLAGS+= -I. -I./common -DXXH_NAMESPACE=ZSTD_ +ifeq ($(OS),Windows_NT) # MinGW assumed +CPPFLAGS += -D__USE_MINGW_ANSI_STDIO # compatibility with %zu formatting +endif CFLAGS ?= -O3 DEBUGFLAGS = -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \ -Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \ @@ -52,11 +55,11 @@ ifeq ($(ZSTD_LIB_DECOMPRESSION), 0) endif ifneq ($(ZSTD_LIB_COMPRESSION), 0) - ZSTD_FILES += $(ZSTDCOMP_FILES) + ZSTD_FILES += $(ZSTDCOMP_FILES) endif ifneq ($(ZSTD_LIB_DECOMPRESSION), 0) - ZSTD_FILES += $(ZSTDDECOMP_FILES) + ZSTD_FILES += $(ZSTDDECOMP_FILES) endif ifneq ($(ZSTD_LIB_DEPRECATED), 0) diff --git a/programs/Makefile b/programs/Makefile index 4202764c2..912f9eff0 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -38,6 +38,9 @@ endif CPPFLAGS+= -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(ZSTDDIR)/compress \ -I$(ZSTDDIR)/dictBuilder \ -DXXH_NAMESPACE=ZSTD_ +ifeq ($(OS),Windows_NT) # MinGW assumed +CPPFLAGS += -D__USE_MINGW_ANSI_STDIO # compatibility with %zu formatting +endif CFLAGS ?= -O3 DEBUGFLAGS+=-Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \ -Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \ @@ -158,7 +161,7 @@ zstd-release: DEBUGFLAGS := zstd-release: zstd zstd32 : CPPFLAGS += $(THREAD_CPP) -zstd32 : LDFLAGS += $(THREAD_LD) +zstd32 : LDFLAGS += $(THREAD_LD) zstd32 : CPPFLAGS += -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT) zstd32 : $(ZSTDLIB_FILES) zstdcli.c fileio.c bench.c datagen.c dibio.c ifneq (,$(filter Windows%,$(OS))) diff --git a/tests/Makefile b/tests/Makefile index 0955272ef..81e685780 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -28,7 +28,7 @@ DEBUGFLAGS = -g -DDEBUGLEVEL=$(DEBUGLEVEL) CPPFLAGS += -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(ZSTDDIR)/compress \ -I$(ZSTDDIR)/dictBuilder -I$(ZSTDDIR)/deprecated -I$(PRGDIR) ifeq ($(OS),Windows_NT) # MinGW assumed -CPPFLAGS += -D__USE_MINGW_ANSI_STDIO +CPPFLAGS += -D__USE_MINGW_ANSI_STDIO # compatibility with %zu formatting endif CFLAGS ?= -O3 CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \ From e85b4c56b288a3480e10117b9e37d6744b665fe9 Mon Sep 17 00:00:00 2001 From: cyan4973 Date: Mon, 30 Jul 2018 17:08:40 +0200 Subject: [PATCH 090/372] speed up appveyor tests --- appveyor.yml | 4 ++-- build/cmake/lib/CMakeLists.txt | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index c9f88dce6..2b674ce3c 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -185,11 +185,11 @@ - COMPILER: "gcc" HOST: "mingw" PLATFORM: "x86" - SCRIPT: "make allzstd" + SCRIPT: "CFLAGS=-Werror make -j allzstd" - COMPILER: "clang" HOST: "mingw" PLATFORM: "x64" - SCRIPT: "MOREFLAGS='--target=x86_64-w64-mingw32 -Werror -Wconversion -Wno-sign-conversion' make allzstd" + SCRIPT: "CFLAGS='--target=x86_64-w64-mingw32 -Werror -Wconversion -Wno-sign-conversion' make -j allzstd" - COMPILER: "visual" HOST: "visual" diff --git a/build/cmake/lib/CMakeLists.txt b/build/cmake/lib/CMakeLists.txt index c4c2f81e6..e84e06301 100644 --- a/build/cmake/lib/CMakeLists.txt +++ b/build/cmake/lib/CMakeLists.txt @@ -14,7 +14,7 @@ OPTION(ZSTD_BUILD_STATIC "BUILD STATIC LIBRARIES" ON) OPTION(ZSTD_BUILD_SHARED "BUILD SHARED LIBRARIES" ON) IF(NOT ZSTD_BUILD_SHARED AND NOT ZSTD_BUILD_STATIC) - MESSAGE(SEND_ERROR "You need to build at least one flavor of libstd") + MESSAGE(SEND_ERROR "You need to build at least one flavor of libzstd") ENDIF() # Define library directory, where sources and header files are located From 31229e527bfa3ae9f0a5e81ed7e00a56dfd494bf Mon Sep 17 00:00:00 2001 From: Jennifer Liu Date: Mon, 30 Jul 2018 12:54:22 -0700 Subject: [PATCH 091/372] Increment frequency for every dmer occurence within same sample instead of at most once per sample --- .../benchmarkDictBuilder/README.md | 200 +++++++++--------- .../fastCover/fastCover.c | 10 +- 2 files changed, 101 insertions(+), 109 deletions(-) diff --git a/contrib/experimental_dict_builders/benchmarkDictBuilder/README.md b/contrib/experimental_dict_builders/benchmarkDictBuilder/README.md index 20fbde954..1fdd323c2 100644 --- a/contrib/experimental_dict_builders/benchmarkDictBuilder/README.md +++ b/contrib/experimental_dict_builders/benchmarkDictBuilder/README.md @@ -18,109 +18,109 @@ make ARG="in=../../../lib/dictBuilder in=../../../lib/compress" - Fourth column is chosen d and fifth column is chosen k github: -NODICT 0.000005 2.999642 -RANDOM 0.141553 8.786957 -LEGACY 0.904340 8.989482 -COVER 53.621302 10.641263 8 1298 -COVER 4.085037 10.641263 8 1298 -FAST15 17.636211 10.586461 8 1778 -FAST15 0.221236 10.586461 8 1778 -FAST16 18.716259 10.492503 6 1778 -FAST16 0.251522 10.492503 6 1778 -FAST17 17.614391 10.611737 8 1778 -FAST17 0.241011 10.611737 8 1778 -FAST18 19.926270 10.621586 8 1778 -FAST18 0.287195 10.621586 8 1778 -FAST19 19.626808 10.629626 8 1778 -FAST19 0.340191 10.629626 8 1778 -FAST20 18.918657 10.610308 8 1778 -FAST20 0.463307 10.610308 8 1778 -FAST21 20.502362 10.625733 8 1778 -FAST21 0.638202 10.625733 8 1778 -FAST22 22.702695 10.625281 8 1778 -FAST22 1.353399 10.625281 8 1778 -FAST23 28.041990 10.602342 8 1778 -FAST23 3.029502 10.602342 8 1778 -FAST24 35.662961 10.603379 8 1778 -FAST24 6.524258 10.603379 8 1778 +NODICT 0.000004 2.999642 +RANDOM 0.161907 8.786957 +LEGACY 0.960128 8.989482 +COVER 69.031037 10.641263 8 1298 +COVER 7.017782 10.641263 8 1298 +FAST15 24.710713 10.547583 8 1874 +FAST15 0.271657 10.547583 8 1874 +FAST16 23.906902 10.690723 8 1106 +FAST16 0.315039 10.690723 8 1106 +FAST17 25.384572 10.642322 8 1106 +FAST17 0.319237 10.642322 8 1106 +FAST18 21.935494 10.491283 8 1826 +FAST18 0.255488 10.491283 8 1826 +FAST19 21.349385 10.522182 8 1826 +FAST19 0.311369 10.522182 8 1826 +FAST20 23.124955 10.487431 8 1826 +FAST20 0.317411 10.487431 8 1826 +FAST21 27.311387 10.491047 8 1778 +FAST21 0.398483 10.491047 8 1778 +FAST22 23.993620 10.502191 8 1826 +FAST22 0.329767 10.502191 8 1826 +FAST23 27.793381 10.502191 8 1826 +FAST23 0.359659 10.502191 8 1826 +FAST24 29.281399 10.509461 8 1826 +FAST24 0.398369 10.509461 8 1826 hg-commands: -NODICT 0.000005 2.425291 -RANDOM 0.080469 3.489515 -LEGACY 0.794417 3.911896 -COVER 54.198788 4.131136 8 386 -COVER 2.191729 4.131136 8 386 -FAST15 11.852793 3.903719 6 1106 -FAST15 0.175406 3.903719 6 1106 -FAST16 12.863315 4.005077 8 530 -FAST16 0.158410 4.005077 8 530 -FAST17 11.977917 4.097811 8 818 -FAST17 0.162381 4.097811 8 818 -FAST18 11.749304 4.136081 8 770 -FAST18 0.173242 4.136081 8 770 -FAST19 11.905785 4.166021 8 530 -FAST19 0.186403 4.166021 8 530 -FAST20 13.293999 4.163740 8 482 -FAST20 0.241508 4.163740 8 482 -FAST21 16.623177 4.157057 8 434 -FAST21 0.372647 4.157057 8 434 -FAST22 20.918409 4.158195 8 290 -FAST22 0.570431 4.158195 8 290 -FAST23 21.762805 4.161450 8 434 -FAST23 1.162206 4.161450 8 434 -FAST24 29.133745 4.159658 8 338 -FAST24 3.054376 4.159658 8 338 +NODICT 0.000007 2.425291 +RANDOM 0.083477 3.489515 +LEGACY 0.941867 3.911896 +COVER 67.314295 4.131136 8 386 +COVER 2.757895 4.131136 8 386 +FAST15 13.466983 3.920128 6 1106 +FAST15 0.162656 3.920128 6 1106 +FAST16 12.618110 4.032422 8 674 +FAST16 0.159073 4.032422 8 674 +FAST17 12.883772 4.063581 8 1490 +FAST17 0.183131 4.063581 8 1490 +FAST18 13.904432 4.085034 8 290 +FAST18 0.161078 4.085034 8 290 +FAST19 13.762269 4.097054 8 578 +FAST19 0.179906 4.097054 8 578 +FAST20 15.303927 4.101575 8 434 +FAST20 0.213146 4.101575 8 434 +FAST21 19.619482 4.104879 8 530 +FAST21 0.289158 4.104879 8 530 +FAST22 23.187937 4.102448 8 530 +FAST22 0.335220 4.102448 8 530 +FAST23 24.946655 4.095162 8 914 +FAST23 0.396927 4.095162 8 914 +FAST24 27.634065 4.114624 8 722 +FAST24 0.434278 4.114624 8 722 hg-changelog: -NODICT 0.000006 1.377613 -RANDOM 0.601346 2.096785 -LEGACY 2.544973 2.058273 -COVER 222.639708 2.188654 8 98 -COVER 6.072892 2.188654 8 98 -FAST15 70.394523 2.127194 8 866 -FAST15 0.899766 2.127194 8 866 -FAST16 69.845529 2.145401 8 338 -FAST16 0.881569 2.145401 8 338 -FAST17 69.382431 2.157544 8 194 -FAST17 0.943291 2.157544 8 194 -FAST18 71.348283 2.173127 8 98 -FAST18 1.034765 2.173127 8 98 -FAST19 71.380923 2.179527 8 98 -FAST19 1.254700 2.179527 8 98 -FAST20 72.802714 2.183233 6 98 -FAST20 1.368704 2.183233 6 98 -FAST21 82.042339 2.180920 8 98 -FAST21 2.213864 2.180920 8 98 -FAST22 90.666200 2.184297 8 98 -FAST22 3.590399 2.184297 8 98 -FAST23 108.926377 2.187666 6 98 -FAST23 8.723759 2.187666 6 98 -FAST24 134.296232 2.189889 6 98 -FAST24 19.396532 2.189889 6 98 +NODICT 0.000027 1.377613 +RANDOM 0.676272 2.096785 +LEGACY 2.871887 2.058273 +COVER 226.371004 2.188654 8 98 +COVER 5.359820 2.188654 8 98 +FAST15 66.776425 2.130548 6 386 +FAST15 0.796836 2.130548 6 386 +FAST16 64.405113 2.144136 8 194 +FAST16 0.778969 2.144136 8 194 +FAST17 65.062292 2.155745 8 98 +FAST17 0.822089 2.155745 8 98 +FAST18 65.819104 2.172062 6 98 +FAST18 0.804247 2.172062 6 98 +FAST19 66.184016 2.179446 6 98 +FAST19 0.883526 2.179446 6 98 +FAST20 72.900924 2.187017 6 98 +FAST20 0.908220 2.187017 6 98 +FAST21 77.869945 2.183583 6 146 +FAST21 0.932666 2.183583 6 146 +FAST22 84.041413 2.182030 6 98 +FAST22 1.092310 2.182030 6 98 +FAST23 89.539265 2.185291 8 98 +FAST23 1.294779 2.185291 8 98 +FAST24 97.193482 2.184939 6 98 +FAST24 1.270493 2.184939 6 98 hg-manifest: -NODICT 0.000005 1.866385 -RANDOM 0.982192 2.309485 -LEGACY 9.507729 2.506775 -COVER 922.742066 2.582597 8 434 -COVER 36.500276 2.582597 8 434 -FAST15 163.886717 2.377689 8 1682 -FAST15 2.107328 2.377689 8 1682 -FAST16 152.684592 2.464814 8 1538 -FAST16 2.157789 2.464814 8 1538 -FAST17 154.463459 2.539834 6 1826 -FAST17 2.282455 2.539834 6 1826 -FAST18 155.540044 2.576924 8 1922 -FAST18 2.101807 2.576924 8 1922 -FAST19 152.650343 2.592479 6 290 -FAST19 2.359461 2.592479 6 290 -FAST20 174.623634 2.594551 8 194 -FAST20 2.870022 2.594551 8 194 -FAST21 219.876653 2.597128 6 194 -FAST21 4.386269 2.597128 6 194 -FAST22 247.986803 2.596971 6 386 -FAST22 6.201144 2.596971 6 386 -FAST23 276.051806 2.601416 8 194 -FAST23 11.613477 2.601416 8 194 -FAST24 328.234024 2.602830 6 194 -FAST24 26.710364 2.602830 6 194 +NODICT 0.000004 1.866385 +RANDOM 0.969045 2.309485 +LEGACY 8.849052 2.506775 +COVER 905.855524 2.582597 8 434 +COVER 34.951973 2.582597 8 434 +FAST15 154.816926 2.391764 6 1826 +FAST15 1.932845 2.391764 6 1826 +FAST16 142.197120 2.480738 6 1922 +FAST16 1.759330 2.480738 6 1922 +FAST17 147.276099 2.548313 6 1682 +FAST17 1.819175 2.548313 6 1682 +FAST18 164.543366 2.567448 6 386 +FAST18 2.728845 2.567448 6 386 +FAST19 195.670852 2.581170 8 338 +FAST19 2.439487 2.581170 8 338 +FAST20 195.716408 2.587062 6 194 +FAST20 2.056303 2.587062 6 194 +FAST21 211.483191 2.590136 6 242 +FAST21 2.983587 2.590136 6 242 +FAST22 239.562966 2.591033 6 194 +FAST22 3.355746 2.591033 6 194 +FAST23 264.547195 2.590403 8 434 +FAST23 3.667851 2.590403 8 434 +FAST24 296.258379 2.591723 6 290 +FAST24 3.858688 2.591723 6 290 diff --git a/contrib/experimental_dict_builders/fastCover/fastCover.c b/contrib/experimental_dict_builders/fastCover/fastCover.c index 3c1aa951c..cf71075ab 100644 --- a/contrib/experimental_dict_builders/fastCover/fastCover.c +++ b/contrib/experimental_dict_builders/fastCover/fastCover.c @@ -266,25 +266,17 @@ static void FASTCOVER_ctx_destroy(FASTCOVER_ctx_t *ctx) { * Calculate for frequency of hash value of each dmer in ctx->samples */ static void FASTCOVER_computeFrequency(U32 *freqs, unsigned f, FASTCOVER_ctx_t *ctx){ - /* inCurrSample keeps track of this hash value has already be seen in previous dmers in the same sample*/ - BYTE* inCurrSample = (BYTE *)malloc((1 << f) * sizeof(BYTE)); size_t start; /* start of current dmer */ for (unsigned i = 0; i < ctx->nbTrainSamples; i++) { - memset(inCurrSample, 0, (1 << f) * sizeof(*inCurrSample)); /* Reset inCurrSample for each sample */ size_t currSampleStart = ctx->offsets[i]; size_t currSampleEnd = ctx->offsets[i+1]; start = currSampleStart; while (start + ctx->d <= currSampleEnd) { const size_t dmerIndex = FASTCOVER_hashPtrToIndex(ctx->samples + start, f, ctx->d); - /* if no dmer with same hash value has been seen in current sample */ - if (inCurrSample[dmerIndex] == 0) { - inCurrSample[dmerIndex]++; - freqs[dmerIndex]++; - } + freqs[dmerIndex]++; start++; } } - free(inCurrSample); } /** From b9faaa1dc3b32b0e6ed248e860db54b4052f0aab Mon Sep 17 00:00:00 2001 From: Nick Terrell Date: Mon, 30 Jul 2018 12:57:11 -0700 Subject: [PATCH 092/372] [FSE] Add division by zero test --- tests/fuzzer.c | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/tests/fuzzer.c b/tests/fuzzer.c index f7bacd5ca..8856a504a 100644 --- a/tests/fuzzer.c +++ b/tests/fuzzer.c @@ -27,6 +27,7 @@ #include /* strcmp */ #include #define ZSTD_STATIC_LINKING_ONLY /* ZSTD_compressContinue, ZSTD_compressBlock */ +#include "fse.h" #include "zstd.h" /* ZSTD_VERSION_STRING */ #include "zstd_errors.h" /* ZSTD_getErrorCode */ #include "zstdmt_compress.h" @@ -1423,6 +1424,24 @@ static int basicUnitTests(U32 seed, double compressibility) } DISPLAYLEVEL(3, "OK \n"); + DISPLAYLEVEL(3, "test%3i : testing FSE_normalizeCount() PR#1255: ", testNb++); + { + short norm[32]; + unsigned count[32]; + unsigned const tableLog = 5; + size_t const nbSeq = 32; + unsigned const maxSymbolValue = 31; + size_t i; + + for (i = 0; i < 32; ++i) + count[i] = 1; + /* Calling FSE_normalizeCount() on a uniform distribution should not + * cause a division by zero. + */ + FSE_normalizeCount(norm, tableLog, count, nbSeq, maxSymbolValue); + } + DISPLAYLEVEL(3, "OK \n"); + _end: free(CNBuffer); free(compressedBuffer); From 4e29bc24699f5e6beb7cff70bc7ffc388bd481e1 Mon Sep 17 00:00:00 2001 From: Jennifer Liu Date: Tue, 31 Jul 2018 10:36:45 -0700 Subject: [PATCH 093/372] Use CDict instead of CCtx in analyzeEntropy --- .../benchmarkDictBuilder/README.md | 200 +++++++++--------- lib/dictBuilder/zdict.c | 32 ++- 2 files changed, 114 insertions(+), 118 deletions(-) diff --git a/contrib/experimental_dict_builders/benchmarkDictBuilder/README.md b/contrib/experimental_dict_builders/benchmarkDictBuilder/README.md index 1fdd323c2..a18311973 100644 --- a/contrib/experimental_dict_builders/benchmarkDictBuilder/README.md +++ b/contrib/experimental_dict_builders/benchmarkDictBuilder/README.md @@ -18,109 +18,109 @@ make ARG="in=../../../lib/dictBuilder in=../../../lib/compress" - Fourth column is chosen d and fifth column is chosen k github: -NODICT 0.000004 2.999642 -RANDOM 0.161907 8.786957 -LEGACY 0.960128 8.989482 -COVER 69.031037 10.641263 8 1298 -COVER 7.017782 10.641263 8 1298 -FAST15 24.710713 10.547583 8 1874 -FAST15 0.271657 10.547583 8 1874 -FAST16 23.906902 10.690723 8 1106 -FAST16 0.315039 10.690723 8 1106 -FAST17 25.384572 10.642322 8 1106 -FAST17 0.319237 10.642322 8 1106 -FAST18 21.935494 10.491283 8 1826 -FAST18 0.255488 10.491283 8 1826 -FAST19 21.349385 10.522182 8 1826 -FAST19 0.311369 10.522182 8 1826 -FAST20 23.124955 10.487431 8 1826 -FAST20 0.317411 10.487431 8 1826 -FAST21 27.311387 10.491047 8 1778 -FAST21 0.398483 10.491047 8 1778 -FAST22 23.993620 10.502191 8 1826 -FAST22 0.329767 10.502191 8 1826 -FAST23 27.793381 10.502191 8 1826 -FAST23 0.359659 10.502191 8 1826 -FAST24 29.281399 10.509461 8 1826 -FAST24 0.398369 10.509461 8 1826 +NODICT 0.000005 2.999642 +RANDOM 0.036114 8.791189 +LEGACY 1.111024 8.173529 +COVER 57.856477 10.652243 8 1298 +COVER 5.769965 10.652243 8 1298 +FAST15 9.965877 10.555630 8 1874 +FAST15 0.140285 10.555630 8 1874 +FAST16 10.337194 10.701698 8 1106 +FAST16 0.114887 10.701698 8 1106 +FAST17 10.207121 10.650652 8 1106 +FAST17 0.135424 10.650652 8 1106 +FAST18 11.463120 10.499142 8 1826 +FAST18 0.154287 10.499142 8 1826 +FAST19 12.143020 10.527140 8 1826 +FAST19 0.158889 10.527140 8 1826 +FAST20 12.510857 10.494710 8 1826 +FAST20 0.171334 10.494710 8 1826 +FAST21 13.201432 10.503488 8 1778 +FAST21 0.192867 10.503488 8 1778 +FAST22 13.754560 10.509284 8 1826 +FAST22 0.206276 10.509284 8 1826 +FAST23 14.708633 10.509284 8 1826 +FAST23 0.221751 10.509284 8 1826 +FAST24 15.134848 10.512369 8 1826 +FAST24 0.234242 10.512369 8 1826 hg-commands: -NODICT 0.000007 2.425291 -RANDOM 0.083477 3.489515 -LEGACY 0.941867 3.911896 -COVER 67.314295 4.131136 8 386 -COVER 2.757895 4.131136 8 386 -FAST15 13.466983 3.920128 6 1106 -FAST15 0.162656 3.920128 6 1106 -FAST16 12.618110 4.032422 8 674 -FAST16 0.159073 4.032422 8 674 -FAST17 12.883772 4.063581 8 1490 -FAST17 0.183131 4.063581 8 1490 -FAST18 13.904432 4.085034 8 290 -FAST18 0.161078 4.085034 8 290 -FAST19 13.762269 4.097054 8 578 -FAST19 0.179906 4.097054 8 578 -FAST20 15.303927 4.101575 8 434 -FAST20 0.213146 4.101575 8 434 -FAST21 19.619482 4.104879 8 530 -FAST21 0.289158 4.104879 8 530 -FAST22 23.187937 4.102448 8 530 -FAST22 0.335220 4.102448 8 530 -FAST23 24.946655 4.095162 8 914 -FAST23 0.396927 4.095162 8 914 -FAST24 27.634065 4.114624 8 722 -FAST24 0.434278 4.114624 8 722 +NODICT 0.000004 2.425291 +RANDOM 0.055073 3.490331 +LEGACY 0.927414 3.911682 +COVER 72.749028 4.132653 8 386 +COVER 3.391066 4.132653 8 386 +FAST15 10.910989 3.920720 6 1106 +FAST15 0.130480 3.920720 6 1106 +FAST16 10.565224 4.033306 8 674 +FAST16 0.146228 4.033306 8 674 +FAST17 11.394137 4.064132 8 1490 +FAST17 0.175567 4.064132 8 1490 +FAST18 11.040248 4.086714 8 290 +FAST18 0.132692 4.086714 8 290 +FAST19 11.335856 4.097947 8 578 +FAST19 0.181441 4.097947 8 578 +FAST20 14.166272 4.102851 8 434 +FAST20 0.203632 4.102851 8 434 +FAST21 15.848896 4.105350 8 530 +FAST21 0.269518 4.105350 8 530 +FAST22 15.570995 4.104100 8 530 +FAST22 0.238512 4.104100 8 530 +FAST23 17.437566 4.098110 8 914 +FAST23 0.270788 4.098110 8 914 +FAST24 18.836604 4.117367 8 722 +FAST24 0.323618 4.117367 8 722 hg-changelog: -NODICT 0.000027 1.377613 -RANDOM 0.676272 2.096785 -LEGACY 2.871887 2.058273 -COVER 226.371004 2.188654 8 98 -COVER 5.359820 2.188654 8 98 -FAST15 66.776425 2.130548 6 386 -FAST15 0.796836 2.130548 6 386 -FAST16 64.405113 2.144136 8 194 -FAST16 0.778969 2.144136 8 194 -FAST17 65.062292 2.155745 8 98 -FAST17 0.822089 2.155745 8 98 -FAST18 65.819104 2.172062 6 98 -FAST18 0.804247 2.172062 6 98 -FAST19 66.184016 2.179446 6 98 -FAST19 0.883526 2.179446 6 98 -FAST20 72.900924 2.187017 6 98 -FAST20 0.908220 2.187017 6 98 -FAST21 77.869945 2.183583 6 146 -FAST21 0.932666 2.183583 6 146 -FAST22 84.041413 2.182030 6 98 -FAST22 1.092310 2.182030 6 98 -FAST23 89.539265 2.185291 8 98 -FAST23 1.294779 2.185291 8 98 -FAST24 97.193482 2.184939 6 98 -FAST24 1.270493 2.184939 6 98 +NODICT 0.000006 1.377613 +RANDOM 0.253393 2.097487 +LEGACY 2.410568 2.058907 +COVER 203.550681 2.189685 8 98 +COVER 7.381697 2.189685 8 98 +FAST15 45.960609 2.130794 6 386 +FAST15 0.512057 2.130794 6 386 +FAST16 44.594817 2.144845 8 194 +FAST16 0.601258 2.144845 8 194 +FAST17 45.852992 2.156099 8 242 +FAST17 0.500844 2.156099 8 242 +FAST18 46.624930 2.172439 6 98 +FAST18 0.680501 2.172439 6 98 +FAST19 47.754905 2.180321 6 98 +FAST19 0.606180 2.180321 6 98 +FAST20 56.733632 2.187431 6 98 +FAST20 0.710149 2.187431 6 98 +FAST21 59.723173 2.184185 6 146 +FAST21 0.875562 2.184185 6 146 +FAST22 66.570788 2.182830 6 98 +FAST22 1.061013 2.182830 6 98 +FAST23 73.817645 2.186399 8 98 +FAST23 0.838496 2.186399 8 98 +FAST24 78.059933 2.185608 6 98 +FAST24 0.843158 2.185608 6 98 hg-manifest: -NODICT 0.000004 1.866385 -RANDOM 0.969045 2.309485 -LEGACY 8.849052 2.506775 -COVER 905.855524 2.582597 8 434 -COVER 34.951973 2.582597 8 434 -FAST15 154.816926 2.391764 6 1826 -FAST15 1.932845 2.391764 6 1826 -FAST16 142.197120 2.480738 6 1922 -FAST16 1.759330 2.480738 6 1922 -FAST17 147.276099 2.548313 6 1682 -FAST17 1.819175 2.548313 6 1682 -FAST18 164.543366 2.567448 6 386 -FAST18 2.728845 2.567448 6 386 -FAST19 195.670852 2.581170 8 338 -FAST19 2.439487 2.581170 8 338 -FAST20 195.716408 2.587062 6 194 -FAST20 2.056303 2.587062 6 194 -FAST21 211.483191 2.590136 6 242 -FAST21 2.983587 2.590136 6 242 -FAST22 239.562966 2.591033 6 194 -FAST22 3.355746 2.591033 6 194 -FAST23 264.547195 2.590403 8 434 -FAST23 3.667851 2.590403 8 434 -FAST24 296.258379 2.591723 6 290 -FAST24 3.858688 2.591723 6 290 +NODICT 0.000005 1.866385 +RANDOM 0.735840 2.309436 +LEGACY 9.322081 2.506977 +COVER 885.961515 2.582528 8 434 +COVER 32.678552 2.582528 8 434 +FAST15 114.414413 2.392920 6 1826 +FAST15 1.412690 2.392920 6 1826 +FAST16 113.869718 2.480762 6 1922 +FAST16 1.539424 2.480762 6 1922 +FAST17 113.333636 2.548285 6 1682 +FAST17 1.473196 2.548285 6 1682 +FAST18 111.717871 2.567634 6 386 +FAST18 1.421200 2.567634 6 386 +FAST19 112.428344 2.581653 8 338 +FAST19 1.412185 2.581653 8 338 +FAST20 128.897480 2.586881 8 194 +FAST20 1.586570 2.586881 8 194 +FAST21 168.465684 2.590051 6 242 +FAST21 2.190732 2.590051 6 242 +FAST22 202.320435 2.591376 6 194 +FAST22 2.667877 2.591376 6 194 +FAST23 228.952201 2.591131 8 434 +FAST23 3.315501 2.591131 8 434 +FAST24 327.320020 2.591548 6 290 +FAST24 5.048348 2.591548 6 290 diff --git a/lib/dictBuilder/zdict.c b/lib/dictBuilder/zdict.c index 2024e0bbb..09b558fea 100644 --- a/lib/dictBuilder/zdict.c +++ b/lib/dictBuilder/zdict.c @@ -581,7 +581,7 @@ static void ZDICT_fillNoise(void* buffer, size_t length) typedef struct { - ZSTD_CCtx* ref; /* contains reference to dictionary */ + ZSTD_CDict* ref; /* contains reference to dictionary */ ZSTD_CCtx* zc; /* working context */ void* workPlace; /* must be ZSTD_BLOCKSIZE_MAX allocated */ } EStats_ress_t; @@ -597,8 +597,9 @@ static void ZDICT_countEStats(EStats_ress_t esr, ZSTD_parameters params, size_t cSize; if (srcSize > blockSizeMax) srcSize = blockSizeMax; /* protection vs large samples */ - { size_t const errorCode = ZSTD_copyCCtx(esr.zc, esr.ref, 0); - if (ZSTD_isError(errorCode)) { DISPLAYLEVEL(1, "warning : ZSTD_copyCCtx failed \n"); return; } + { size_t const errorCode = ZSTD_compressBegin_usingCDict(esr.zc, esr.ref); + if (ZSTD_isError(errorCode)) { DISPLAYLEVEL(1, "warning : ZSTD_compressBegin_usingCDict failed \n"); return; } + } cSize = ZSTD_compressBlock(esr.zc, esr.workPlace, ZSTD_BLOCKSIZE_MAX, src, srcSize); if (ZSTD_isError(cSize)) { DISPLAYLEVEL(3, "warning : could not compress sample size %u \n", (U32)srcSize); return; } @@ -708,14 +709,6 @@ static size_t ZDICT_analyzeEntropy(void* dstBuffer, size_t maxDstSize, /* init */ DEBUGLOG(4, "ZDICT_analyzeEntropy"); - esr.ref = ZSTD_createCCtx(); - esr.zc = ZSTD_createCCtx(); - esr.workPlace = malloc(ZSTD_BLOCKSIZE_MAX); - if (!esr.ref || !esr.zc || !esr.workPlace) { - eSize = ERROR(memory_allocation); - DISPLAYLEVEL(1, "Not enough memory \n"); - goto _cleanup; - } if (offcodeMax>OFFCODE_MAX) { eSize = ERROR(dictionaryCreation_failed); goto _cleanup; } /* too large dictionary */ for (u=0; u<256; u++) countLit[u] = 1; /* any character must be described */ for (u=0; u<=offcodeMax; u++) offcodeCount[u] = 1; @@ -726,12 +719,15 @@ static size_t ZDICT_analyzeEntropy(void* dstBuffer, size_t maxDstSize, memset(bestRepOffset, 0, sizeof(bestRepOffset)); if (compressionLevel==0) compressionLevel = g_compressionLevel_default; params = ZSTD_getParams(compressionLevel, averageSampleSize, dictBufferSize); - { size_t const beginResult = ZSTD_compressBegin_advanced(esr.ref, dictBuffer, dictBufferSize, params, 0); - if (ZSTD_isError(beginResult)) { - DISPLAYLEVEL(1, "error : ZSTD_compressBegin_advanced() failed : %s \n", ZSTD_getErrorName(beginResult)); - eSize = ERROR(GENERIC); - goto _cleanup; - } } + + esr.ref = ZSTD_createCDict_advanced(dictBuffer, dictBufferSize, ZSTD_dlm_byCopy, ZSTD_dct_auto, params.cParams, ZSTD_defaultCMem); + esr.zc = ZSTD_createCCtx(); + esr.workPlace = malloc(ZSTD_BLOCKSIZE_MAX); + if (!esr.ref || !esr.zc || !esr.workPlace) { + eSize = ERROR(memory_allocation); + DISPLAYLEVEL(1, "Not enough memory \n"); + goto _cleanup; + } /* collect stats on all samples */ for (u=0; u Date: Tue, 31 Jul 2018 13:58:54 -0700 Subject: [PATCH 094/372] Refactoring --- lib/dictBuilder/zdict.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/lib/dictBuilder/zdict.c b/lib/dictBuilder/zdict.c index 09b558fea..b8a51789a 100644 --- a/lib/dictBuilder/zdict.c +++ b/lib/dictBuilder/zdict.c @@ -293,7 +293,7 @@ static dictItem ZDICT_analyzePos( refinedEnd = refinedStart + selectedCount; } - /* evaluate gain based on new ref */ + /* evaluate gain based on new dict */ start = refinedStart; pos = suffix[refinedStart]; end = start; @@ -341,7 +341,7 @@ static dictItem ZDICT_analyzePos( for (i=MINMATCHLENGTH; i<=(int)maxLength; i++) savings[i] = savings[i-1] + (lengthList[i] * (i-3)); - DISPLAYLEVEL(4, "Selected ref at position %u, of length %u : saves %u (ratio: %.2f) \n", + DISPLAYLEVEL(4, "Selected dict at position %u, of length %u : saves %u (ratio: %.2f) \n", (U32)pos, (U32)maxLength, savings[maxLength], (double)savings[maxLength] / maxLength); solution.pos = (U32)pos; @@ -581,7 +581,7 @@ static void ZDICT_fillNoise(void* buffer, size_t length) typedef struct { - ZSTD_CDict* ref; /* contains reference to dictionary */ + ZSTD_CDict* dict; /* dictionary */ ZSTD_CCtx* zc; /* working context */ void* workPlace; /* must be ZSTD_BLOCKSIZE_MAX allocated */ } EStats_ress_t; @@ -597,7 +597,7 @@ static void ZDICT_countEStats(EStats_ress_t esr, ZSTD_parameters params, size_t cSize; if (srcSize > blockSizeMax) srcSize = blockSizeMax; /* protection vs large samples */ - { size_t const errorCode = ZSTD_compressBegin_usingCDict(esr.zc, esr.ref); + { size_t const errorCode = ZSTD_compressBegin_usingCDict(esr.zc, esr.dict); if (ZSTD_isError(errorCode)) { DISPLAYLEVEL(1, "warning : ZSTD_compressBegin_usingCDict failed \n"); return; } } @@ -720,10 +720,10 @@ static size_t ZDICT_analyzeEntropy(void* dstBuffer, size_t maxDstSize, if (compressionLevel==0) compressionLevel = g_compressionLevel_default; params = ZSTD_getParams(compressionLevel, averageSampleSize, dictBufferSize); - esr.ref = ZSTD_createCDict_advanced(dictBuffer, dictBufferSize, ZSTD_dlm_byCopy, ZSTD_dct_auto, params.cParams, ZSTD_defaultCMem); + esr.dict = ZSTD_createCDict_advanced(dictBuffer, dictBufferSize, ZSTD_dlm_byRef, ZSTD_dct_rawContent, params.cParams, ZSTD_defaultCMem); esr.zc = ZSTD_createCCtx(); esr.workPlace = malloc(ZSTD_BLOCKSIZE_MAX); - if (!esr.ref || !esr.zc || !esr.workPlace) { + if (!esr.dict || !esr.zc || !esr.workPlace) { eSize = ERROR(memory_allocation); DISPLAYLEVEL(1, "Not enough memory \n"); goto _cleanup; @@ -852,7 +852,7 @@ static size_t ZDICT_analyzeEntropy(void* dstBuffer, size_t maxDstSize, eSize += 12; _cleanup: - ZSTD_freeCDict(esr.ref); + ZSTD_freeCDict(esr.dict); ZSTD_freeCCtx(esr.zc); free(esr.workPlace); From 0acb0abd1e470b8e36ebde1498d79fc6a1990cc5 Mon Sep 17 00:00:00 2001 From: Jennifer Liu Date: Wed, 1 Aug 2018 11:06:16 -0700 Subject: [PATCH 095/372] Add non-optimize FASTCOVER (#1260) * Add non-optimize FASTCOVER * Minor fix * Pass param as value instead of pointer --- .../benchmarkDictBuilder/README.md | 200 +++++++++--------- .../benchmarkDictBuilder/benchmark.c | 22 +- .../fastCover/README.md | 4 +- .../fastCover/fastCover.c | 55 ++++- .../fastCover/fastCover.h | 24 ++- .../fastCover/main.c | 14 +- .../fastCover/test.sh | 8 +- 7 files changed, 201 insertions(+), 126 deletions(-) diff --git a/contrib/experimental_dict_builders/benchmarkDictBuilder/README.md b/contrib/experimental_dict_builders/benchmarkDictBuilder/README.md index a18311973..559776e2b 100644 --- a/contrib/experimental_dict_builders/benchmarkDictBuilder/README.md +++ b/contrib/experimental_dict_builders/benchmarkDictBuilder/README.md @@ -18,109 +18,109 @@ make ARG="in=../../../lib/dictBuilder in=../../../lib/compress" - Fourth column is chosen d and fifth column is chosen k github: -NODICT 0.000005 2.999642 -RANDOM 0.036114 8.791189 -LEGACY 1.111024 8.173529 -COVER 57.856477 10.652243 8 1298 -COVER 5.769965 10.652243 8 1298 -FAST15 9.965877 10.555630 8 1874 -FAST15 0.140285 10.555630 8 1874 -FAST16 10.337194 10.701698 8 1106 -FAST16 0.114887 10.701698 8 1106 -FAST17 10.207121 10.650652 8 1106 -FAST17 0.135424 10.650652 8 1106 -FAST18 11.463120 10.499142 8 1826 -FAST18 0.154287 10.499142 8 1826 -FAST19 12.143020 10.527140 8 1826 -FAST19 0.158889 10.527140 8 1826 -FAST20 12.510857 10.494710 8 1826 -FAST20 0.171334 10.494710 8 1826 -FAST21 13.201432 10.503488 8 1778 -FAST21 0.192867 10.503488 8 1778 -FAST22 13.754560 10.509284 8 1826 -FAST22 0.206276 10.509284 8 1826 -FAST23 14.708633 10.509284 8 1826 -FAST23 0.221751 10.509284 8 1826 -FAST24 15.134848 10.512369 8 1826 -FAST24 0.234242 10.512369 8 1826 +NODICT 0.000025 2.999642 +RANDOM 0.030101 8.791189 +LEGACY 0.913108 8.173529 +COVER 59.234160 10.652243 8 1298 +COVER 6.258459 10.652243 8 1298 +FAST15 9.959246 10.555630 8 1874 +FAST15 0.077719 10.555630 8 1874 +FAST16 10.028343 10.701698 8 1106 +FAST16 0.078117 10.701698 8 1106 +FAST17 10.567355 10.650652 8 1106 +FAST17 0.124833 10.650652 8 1106 +FAST18 11.795287 10.499142 8 1826 +FAST18 0.086992 10.499142 8 1826 +FAST19 13.132451 10.527140 8 1826 +FAST19 0.134716 10.527140 8 1826 +FAST20 14.366314 10.494710 8 1826 +FAST20 0.128844 10.494710 8 1826 +FAST21 14.941238 10.503488 8 1778 +FAST21 0.134975 10.503488 8 1778 +FAST22 15.146226 10.509284 8 1826 +FAST22 0.146918 10.509284 8 1826 +FAST23 16.260552 10.509284 8 1826 +FAST23 0.158494 10.509284 8 1826 +FAST24 16.806037 10.512369 8 1826 +FAST24 0.190464 10.512369 8 1826 hg-commands: -NODICT 0.000004 2.425291 -RANDOM 0.055073 3.490331 -LEGACY 0.927414 3.911682 -COVER 72.749028 4.132653 8 386 -COVER 3.391066 4.132653 8 386 -FAST15 10.910989 3.920720 6 1106 -FAST15 0.130480 3.920720 6 1106 -FAST16 10.565224 4.033306 8 674 -FAST16 0.146228 4.033306 8 674 -FAST17 11.394137 4.064132 8 1490 -FAST17 0.175567 4.064132 8 1490 -FAST18 11.040248 4.086714 8 290 -FAST18 0.132692 4.086714 8 290 -FAST19 11.335856 4.097947 8 578 -FAST19 0.181441 4.097947 8 578 -FAST20 14.166272 4.102851 8 434 -FAST20 0.203632 4.102851 8 434 -FAST21 15.848896 4.105350 8 530 -FAST21 0.269518 4.105350 8 530 -FAST22 15.570995 4.104100 8 530 -FAST22 0.238512 4.104100 8 530 -FAST23 17.437566 4.098110 8 914 -FAST23 0.270788 4.098110 8 914 -FAST24 18.836604 4.117367 8 722 -FAST24 0.323618 4.117367 8 722 +NODICT 0.000026 2.425291 +RANDOM 0.046270 3.490331 +LEGACY 0.847904 3.911682 +COVER 71.691804 4.132653 8 386 +COVER 3.187085 4.132653 8 386 +FAST15 11.593687 3.920720 6 1106 +FAST15 0.082431 3.920720 6 1106 +FAST16 11.775958 4.033306 8 674 +FAST16 0.092587 4.033306 8 674 +FAST17 11.965064 4.064132 8 1490 +FAST17 0.106382 4.064132 8 1490 +FAST18 11.438197 4.086714 8 290 +FAST18 0.097293 4.086714 8 290 +FAST19 12.292512 4.097947 8 578 +FAST19 0.104406 4.097947 8 578 +FAST20 13.857857 4.102851 8 434 +FAST20 0.139467 4.102851 8 434 +FAST21 14.599613 4.105350 8 530 +FAST21 0.189416 4.105350 8 530 +FAST22 15.966109 4.104100 8 530 +FAST22 0.183817 4.104100 8 530 +FAST23 18.033645 4.098110 8 914 +FAST23 0.246641 4.098110 8 914 +FAST24 22.992891 4.117367 8 722 +FAST24 0.285994 4.117367 8 722 hg-changelog: -NODICT 0.000006 1.377613 -RANDOM 0.253393 2.097487 -LEGACY 2.410568 2.058907 -COVER 203.550681 2.189685 8 98 -COVER 7.381697 2.189685 8 98 -FAST15 45.960609 2.130794 6 386 -FAST15 0.512057 2.130794 6 386 -FAST16 44.594817 2.144845 8 194 -FAST16 0.601258 2.144845 8 194 -FAST17 45.852992 2.156099 8 242 -FAST17 0.500844 2.156099 8 242 -FAST18 46.624930 2.172439 6 98 -FAST18 0.680501 2.172439 6 98 -FAST19 47.754905 2.180321 6 98 -FAST19 0.606180 2.180321 6 98 -FAST20 56.733632 2.187431 6 98 -FAST20 0.710149 2.187431 6 98 -FAST21 59.723173 2.184185 6 146 -FAST21 0.875562 2.184185 6 146 -FAST22 66.570788 2.182830 6 98 -FAST22 1.061013 2.182830 6 98 -FAST23 73.817645 2.186399 8 98 -FAST23 0.838496 2.186399 8 98 -FAST24 78.059933 2.185608 6 98 -FAST24 0.843158 2.185608 6 98 +NODICT 0.000007 1.377613 +RANDOM 0.297345 2.097487 +LEGACY 2.633992 2.058907 +COVER 219.179786 2.189685 8 98 +COVER 6.620852 2.189685 8 98 +FAST15 47.635082 2.130794 6 386 +FAST15 0.321297 2.130794 6 386 +FAST16 43.837676 2.144845 8 194 +FAST16 0.312640 2.144845 8 194 +FAST17 49.349017 2.156099 8 242 +FAST17 0.348459 2.156099 8 242 +FAST18 51.153784 2.172439 6 98 +FAST18 0.353106 2.172439 6 98 +FAST19 52.627045 2.180321 6 98 +FAST19 0.390612 2.180321 6 98 +FAST20 63.748782 2.187431 6 98 +FAST20 0.489544 2.187431 6 98 +FAST21 68.709198 2.184185 6 146 +FAST21 0.530852 2.184185 6 146 +FAST22 68.491639 2.182830 6 98 +FAST22 0.645699 2.182830 6 98 +FAST23 72.558688 2.186399 8 98 +FAST23 0.593539 2.186399 8 98 +FAST24 76.137195 2.185608 6 98 +FAST24 0.680132 2.185608 6 98 hg-manifest: -NODICT 0.000005 1.866385 -RANDOM 0.735840 2.309436 -LEGACY 9.322081 2.506977 -COVER 885.961515 2.582528 8 434 -COVER 32.678552 2.582528 8 434 -FAST15 114.414413 2.392920 6 1826 -FAST15 1.412690 2.392920 6 1826 -FAST16 113.869718 2.480762 6 1922 -FAST16 1.539424 2.480762 6 1922 -FAST17 113.333636 2.548285 6 1682 -FAST17 1.473196 2.548285 6 1682 -FAST18 111.717871 2.567634 6 386 -FAST18 1.421200 2.567634 6 386 -FAST19 112.428344 2.581653 8 338 -FAST19 1.412185 2.581653 8 338 -FAST20 128.897480 2.586881 8 194 -FAST20 1.586570 2.586881 8 194 -FAST21 168.465684 2.590051 6 242 -FAST21 2.190732 2.590051 6 242 -FAST22 202.320435 2.591376 6 194 -FAST22 2.667877 2.591376 6 194 -FAST23 228.952201 2.591131 8 434 -FAST23 3.315501 2.591131 8 434 -FAST24 327.320020 2.591548 6 290 -FAST24 5.048348 2.591548 6 290 +NODICT 0.000026 1.866385 +RANDOM 0.784554 2.309436 +LEGACY 10.193714 2.506977 +COVER 988.206583 2.582528 8 434 +COVER 39.726199 2.582528 8 434 +FAST15 168.388819 2.392920 6 1826 +FAST15 1.272178 2.392920 6 1826 +FAST16 161.822607 2.480762 6 1922 +FAST16 1.164908 2.480762 6 1922 +FAST17 157.688544 2.548285 6 1682 +FAST17 1.222439 2.548285 6 1682 +FAST18 154.529585 2.567634 6 386 +FAST18 1.217596 2.567634 6 386 +FAST19 160.244979 2.581653 8 338 +FAST19 1.282450 2.581653 8 338 +FAST20 191.503297 2.586881 8 194 +FAST20 2.009748 2.586881 8 194 +FAST21 226.389709 2.590051 6 242 +FAST21 2.494543 2.590051 6 242 +FAST22 217.859055 2.591376 6 194 +FAST22 2.295693 2.591376 6 194 +FAST23 236.819791 2.591131 8 434 +FAST23 2.744711 2.591131 8 434 +FAST24 269.187800 2.591548 6 290 +FAST24 2.923671 2.591548 6 290 diff --git a/contrib/experimental_dict_builders/benchmarkDictBuilder/benchmark.c b/contrib/experimental_dict_builders/benchmarkDictBuilder/benchmark.c index 75008a087..d92e8d5cb 100644 --- a/contrib/experimental_dict_builders/benchmarkDictBuilder/benchmark.c +++ b/contrib/experimental_dict_builders/benchmarkDictBuilder/benchmark.c @@ -91,14 +91,26 @@ dictInfo* createDictFromFiles(sampleInfo *info, unsigned maxDictSize, dictSize = ZDICT_trainFromBuffer_random(dictBuffer, maxDictSize, info->srcBuffer, info->samplesSizes, info->nbSamples, *randomParams); }else if(coverParams) { - dictSize = ZDICT_optimizeTrainFromBuffer_cover(dictBuffer, maxDictSize, info->srcBuffer, - info->samplesSizes, info->nbSamples, coverParams); + /* Run the optimize version if either k or d is not provided */ + if (!coverParams->d || !coverParams->k){ + dictSize = ZDICT_optimizeTrainFromBuffer_cover(dictBuffer, maxDictSize, info->srcBuffer, + info->samplesSizes, info->nbSamples, coverParams); + } else { + dictSize = ZDICT_trainFromBuffer_cover(dictBuffer, maxDictSize, info->srcBuffer, + info->samplesSizes, info->nbSamples, *coverParams); + } } else if(legacyParams) { dictSize = ZDICT_trainFromBuffer_legacy(dictBuffer, maxDictSize, info->srcBuffer, info->samplesSizes, info->nbSamples, *legacyParams); } else if(fastParams) { - dictSize = ZDICT_optimizeTrainFromBuffer_fastCover(dictBuffer, maxDictSize, info->srcBuffer, - info->samplesSizes, info->nbSamples, fastParams); + /* Run the optimize version if either k or d is not provided */ + if (!fastParams->d || !fastParams->k) { + dictSize = ZDICT_optimizeTrainFromBuffer_fastCover(dictBuffer, maxDictSize, info->srcBuffer, + info->samplesSizes, info->nbSamples, fastParams); + } else { + dictSize = ZDICT_trainFromBuffer_fastCover(dictBuffer, maxDictSize, info->srcBuffer, + info->samplesSizes, info->nbSamples, *fastParams); + } } else { dictSize = 0; } @@ -403,7 +415,6 @@ int main(int argCount, const char* argv[]) goto _cleanup; } - /* for fastCover (with k and d provided) */ const int fastResult = benchmarkDictBuilder(srcInfo, maxDictSize, NULL, NULL, NULL, &fastParam); DISPLAYLEVEL(2, "k=%u\nd=%u\nf=%u\nsteps=%u\nsplit=%u\n", fastParam.k, fastParam.d, fastParam.f, fastParam.steps, (unsigned)(fastParam.splitPoint * 100)); @@ -411,7 +422,6 @@ int main(int argCount, const char* argv[]) result = 1; goto _cleanup; } - } diff --git a/contrib/experimental_dict_builders/fastCover/README.md b/contrib/experimental_dict_builders/fastCover/README.md index 66e00ee04..ad377743f 100644 --- a/contrib/experimental_dict_builders/fastCover/README.md +++ b/contrib/experimental_dict_builders/fastCover/README.md @@ -16,8 +16,8 @@ make test ###Usage: -To build a random dictionary with the provided arguments: make ARG= followed by arguments - +To build a FASTCOVER dictionary with the provided arguments: make ARG= followed by arguments +If k or d is not provided, the optimize version of FASTCOVER is run. ### Examples: make ARG="in=../../../lib/dictBuilder out=dict100 dictID=520" diff --git a/contrib/experimental_dict_builders/fastCover/fastCover.c b/contrib/experimental_dict_builders/fastCover/fastCover.c index cf71075ab..84d841b10 100644 --- a/contrib/experimental_dict_builders/fastCover/fastCover.c +++ b/contrib/experimental_dict_builders/fastCover/fastCover.c @@ -629,6 +629,55 @@ _cleanup: } } +ZDICTLIB_API size_t ZDICT_trainFromBuffer_fastCover( + void *dictBuffer, size_t dictBufferCapacity, const void *samplesBuffer, + const size_t *samplesSizes, unsigned nbSamples, ZDICT_fastCover_params_t parameters) { + BYTE* const dict = (BYTE*)dictBuffer; + FASTCOVER_ctx_t ctx; + parameters.splitPoint = 1.0; + /* Initialize global data */ + g_displayLevel = parameters.zParams.notificationLevel; + /* Checks */ + if (!FASTCOVER_checkParameters(parameters, dictBufferCapacity)) { + DISPLAYLEVEL(1, "FASTCOVER parameters incorrect\n"); + return ERROR(GENERIC); + } + if (nbSamples == 0) { + DISPLAYLEVEL(1, "FASTCOVER must have at least one input file\n"); + return ERROR(GENERIC); + } + if (dictBufferCapacity < ZDICT_DICTSIZE_MIN) { + DISPLAYLEVEL(1, "dictBufferCapacity must be at least %u\n", + ZDICT_DICTSIZE_MIN); + return ERROR(dstSize_tooSmall); + } + /* Initialize context */ + if (!FASTCOVER_ctx_init(&ctx, samplesBuffer, samplesSizes, nbSamples, + parameters.d, parameters.splitPoint, parameters.f)) { + DISPLAYLEVEL(1, "Failed to initialize context\n"); + return ERROR(GENERIC); + } + /* Build the dictionary */ + DISPLAYLEVEL(2, "Building dictionary\n"); + { + const size_t tail = FASTCOVER_buildDictionary(&ctx, ctx.freqs, dictBuffer, + dictBufferCapacity, parameters); + + const size_t dictionarySize = ZDICT_finalizeDictionary( + dict, dictBufferCapacity, dict + tail, dictBufferCapacity - tail, + samplesBuffer, samplesSizes, (unsigned)ctx.nbTrainSamples, + parameters.zParams); + if (!ZSTD_isError(dictionarySize)) { + DISPLAYLEVEL(2, "Constructed dictionary of size %u\n", + (U32)dictionarySize); + } + FASTCOVER_ctx_destroy(&ctx); + return dictionarySize; + } +} + + + ZDICTLIB_API size_t ZDICT_optimizeTrainFromBuffer_fastCover( void *dictBuffer, size_t dictBufferCapacity, const void *samplesBuffer, const size_t *samplesSizes, unsigned nbSamples, @@ -657,15 +706,15 @@ ZDICTLIB_API size_t ZDICT_optimizeTrainFromBuffer_fastCover( /* Checks */ if (splitPoint <= 0 || splitPoint > 1) { - LOCALDISPLAYLEVEL(displayLevel, 1, "Incorrect parameters\n"); + LOCALDISPLAYLEVEL(displayLevel, 1, "Incorrect splitPoint\n"); return ERROR(GENERIC); } if (kMinK < kMaxD || kMaxK < kMinK) { - LOCALDISPLAYLEVEL(displayLevel, 1, "Incorrect parameters\n"); + LOCALDISPLAYLEVEL(displayLevel, 1, "Incorrect k\n"); return ERROR(GENERIC); } if (nbSamples == 0) { - DISPLAYLEVEL(1, "fast must have at least one input file\n"); + DISPLAYLEVEL(1, "FASTCOVER must have at least one input file\n"); return ERROR(GENERIC); } if (dictBufferCapacity < ZDICT_DICTSIZE_MIN) { diff --git a/contrib/experimental_dict_builders/fastCover/fastCover.h b/contrib/experimental_dict_builders/fastCover/fastCover.h index eca04baab..958e9f423 100644 --- a/contrib/experimental_dict_builders/fastCover/fastCover.h +++ b/contrib/experimental_dict_builders/fastCover/fastCover.h @@ -12,9 +12,6 @@ #include "zdict.h" - - - typedef struct { unsigned k; /* Segment size : constraint: 0 < k : Reasonable range [16, 2048+] */ unsigned d; /* dmer size : constraint: 0 < d <= k : Reasonable range [6, 16] */ @@ -26,7 +23,6 @@ typedef struct { } ZDICT_fastCover_params_t; - /*! ZDICT_optimizeTrainFromBuffer_fastCover(): * Train a dictionary from an array of samples using a modified version of the COVER algorithm. * Samples must be stored concatenated in a single flat buffer `samplesBuffer`, @@ -41,7 +37,21 @@ typedef struct { * or an error code, which can be tested with ZDICT_isError(). * On success `*parameters` contains the parameters selected. */ -ZDICTLIB_API size_t ZDICT_optimizeTrainFromBuffer_fastCover( + ZDICTLIB_API size_t ZDICT_optimizeTrainFromBuffer_fastCover( + void *dictBuffer, size_t dictBufferCapacity, const void *samplesBuffer, + const size_t *samplesSizes, unsigned nbSamples, + ZDICT_fastCover_params_t *parameters); + + +/*! ZDICT_trainFromBuffer_fastCover(): + * Train a dictionary from an array of samples using a modified version of the COVER algorithm. + * Samples must be stored concatenated in a single flat buffer `samplesBuffer`, + * supplied with an array of sizes `samplesSizes`, providing the size of each sample, in order. + * The resulting dictionary will be saved into `dictBuffer`. + * d, k, and f are required. + * @return: size of dictionary stored into `dictBuffer` (<= `dictBufferCapacity`) + * or an error code, which can be tested with ZDICT_isError(). + */ +ZDICTLIB_API size_t ZDICT_trainFromBuffer_fastCover( void *dictBuffer, size_t dictBufferCapacity, const void *samplesBuffer, - const size_t *samplesSizes, unsigned nbSamples, - ZDICT_fastCover_params_t *parameters); + const size_t *samplesSizes, unsigned nbSamples, ZDICT_fastCover_params_t parameters); diff --git a/contrib/experimental_dict_builders/fastCover/main.c b/contrib/experimental_dict_builders/fastCover/main.c index f286b0506..df7d91812 100644 --- a/contrib/experimental_dict_builders/fastCover/main.c +++ b/contrib/experimental_dict_builders/fastCover/main.c @@ -64,8 +64,14 @@ int FASTCOVER_trainFromFiles(const char* dictFileName, sampleInfo *info, EXM_THROW(12, "not enough memory for trainFromFiles"); /* should not happen */ { size_t dictSize; - dictSize = ZDICT_optimizeTrainFromBuffer_fastCover(dictBuffer, maxDictSize, info->srcBuffer, - info->samplesSizes, info->nbSamples, params); + /* Run the optimize version if either k or d is not provided */ + if (!params->d || !params->k) { + dictSize = ZDICT_optimizeTrainFromBuffer_fastCover(dictBuffer, maxDictSize, info->srcBuffer, + info->samplesSizes, info->nbSamples, params); + } else { + dictSize = ZDICT_trainFromBuffer_fastCover(dictBuffer, maxDictSize, info->srcBuffer, + info->samplesSizes, info->nbSamples, *params); + } DISPLAYLEVEL(2, "k=%u\nd=%u\nf=%u\nsteps=%u\nsplit=%u\n", params->k, params->d, params->f, params->steps, (unsigned)(params->splitPoint*100)); if (ZDICT_isError(dictSize)) { DISPLAYLEVEL(1, "dictionary training failed : %s \n", ZDICT_getErrorName(dictSize)); /* should not happen */ @@ -92,8 +98,8 @@ int main(int argCount, const char* argv[]) int operationResult = 0; /* Initialize arguments to default values */ - unsigned k = 200; - unsigned d = 8; + unsigned k = 0; + unsigned d = 0; unsigned f = 23; unsigned steps = 32; unsigned nbThreads = 1; diff --git a/contrib/experimental_dict_builders/fastCover/test.sh b/contrib/experimental_dict_builders/fastCover/test.sh index 91d4f4923..f86915b59 100644 --- a/contrib/experimental_dict_builders/fastCover/test.sh +++ b/contrib/experimental_dict_builders/fastCover/test.sh @@ -1,8 +1,8 @@ -echo "Building fastCover dictionary with in=../../lib/common k=200 f=20 out=dict1" -./main in=../../../lib/common k=200 f=20 out=dict1 +echo "Building fastCover dictionary with in=../../lib/common f=20 out=dict1" +./main in=../../../lib/common f=20 out=dict1 zstd -be3 -D dict1 -r ../../../lib/common -q -echo "Building fastCover dictionary with in=../../lib/common k=500 f=24 out=dict2 dictID=100 maxdict=140000" -./main in=../../../lib/common k=500 f=24 out=dict2 dictID=100 maxdict=140000 +echo "Building fastCover dictionary with in=../../lib/common k=500 d=6 f=24 out=dict2 dictID=100 maxdict=140000" +./main in=../../../lib/common k=500 d=6 f=24 out=dict2 dictID=100 maxdict=140000 zstd -be3 -D dict2 -r ../../../lib/common -q echo "Building fastCover dictionary with 2 sample sources" ./main in=../../../lib/common in=../../../lib/compress out=dict3 From dc5a67cb7b84a86bb3729ae4a85ef61e37797ac2 Mon Sep 17 00:00:00 2001 From: Nick Terrell Date: Thu, 2 Aug 2018 11:12:17 -0700 Subject: [PATCH 096/372] Disallow tableLog == srcLog --- lib/compress/fse_compress.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/compress/fse_compress.c b/lib/compress/fse_compress.c index 95e7c1c7e..70daae3bc 100644 --- a/lib/compress/fse_compress.c +++ b/lib/compress/fse_compress.c @@ -331,7 +331,7 @@ void FSE_freeCTable (FSE_CTable* ct) { free(ct); } /* provides the minimum logSize to safely represent a distribution */ static unsigned FSE_minTableLog(size_t srcSize, unsigned maxSymbolValue) { - U32 minBitsSrc = BIT_highbit32((U32)(srcSize - 1)) + 1; + U32 minBitsSrc = BIT_highbit32((U32)(srcSize)) + 1; U32 minBitsSymbols = BIT_highbit32(maxSymbolValue) + 2; U32 minBits = minBitsSrc < minBitsSymbols ? minBitsSrc : minBitsSymbols; assert(srcSize > 1); /* Not supported, RLE should be used instead */ From ab6e038482f78d5c3a31a0e74fc0856e1ee91bf5 Mon Sep 17 00:00:00 2001 From: Jennifer Liu Date: Wed, 1 Aug 2018 17:39:15 -0700 Subject: [PATCH 097/372] Minor fix --- contrib/experimental_dict_builders/fastCover/fastCover.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/contrib/experimental_dict_builders/fastCover/fastCover.c b/contrib/experimental_dict_builders/fastCover/fastCover.c index 84d841b10..02c155a81 100644 --- a/contrib/experimental_dict_builders/fastCover/fastCover.c +++ b/contrib/experimental_dict_builders/fastCover/fastCover.c @@ -197,7 +197,7 @@ static FASTCOVER_segment_t FASTCOVER_selectSegment(const FASTCOVER_ctx_t *ctx, bestSegment.end = newEnd; } { - /* Half the frequency of hash value of each dmer covered by the chosen segment. */ + /* Zero the frequency of hash value of each dmer covered by the chosen segment. */ U32 pos; for (pos = bestSegment.begin; pos != bestSegment.end; ++pos) { const size_t i = FASTCOVER_hashPtrToIndex(ctx->samples + pos, parameters.f, ctx->d); @@ -300,7 +300,7 @@ static int FASTCOVER_ctx_init(FASTCOVER_ctx_t *ctx, const void *samplesBuffer, if (totalSamplesSize < MAX(d, sizeof(U64)) || totalSamplesSize >= (size_t)FASTCOVER_MAX_SAMPLES_SIZE) { DISPLAYLEVEL(1, "Total samples size is too large (%u MB), maximum size is %u MB\n", - (U32)(totalSamplesSize>>20), (FASTCOVER_MAX_SAMPLES_SIZE >> 20)); + (U32)(totalSamplesSize >> 20), (FASTCOVER_MAX_SAMPLES_SIZE >> 20)); return 0; } /* Check if there are at least 5 training samples */ From 5203f01774397f4e6265d0ab3300805f8f4bc6fb Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Fri, 3 Aug 2018 07:54:29 -0700 Subject: [PATCH 098/372] fix : zstd cli can be built with build macro ZSTD_NOBENCH which disables bench.c module --- Makefile | 2 +- programs/zstdcli.c | 32 +++++++++++++++++--------------- 2 files changed, 18 insertions(+), 16 deletions(-) diff --git a/Makefile b/Makefile index 89c5a418d..98c35a346 100644 --- a/Makefile +++ b/Makefile @@ -65,7 +65,7 @@ zlibwrapper: .PHONY: test test: MOREFLAGS += -g -DDEBUGLEVEL=1 -Werror test: - MOREFLAGS="$(MOREFLAGS)" $(MAKE) -C $(PRGDIR) allVariants + MOREFLAGS="$(MOREFLAGS)" $(MAKE) -j -C $(PRGDIR) allVariants $(MAKE) -C $(TESTDIR) $@ .PHONY: shortest diff --git a/programs/zstdcli.c b/programs/zstdcli.c index 36ba21156..34f73302a 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -406,7 +406,6 @@ int main(int argCount, const char* argv[]) singleThread = 0, ultra=0; double compressibility = 0.5; - BMK_advancedParams_t adv = BMK_initAdvancedParams(); unsigned bench_nbSeconds = 3; /* would be better if this value was synchronized from bench */ size_t blockSize = 0; zstd_operation_mode operation = zom_compress; @@ -434,6 +433,9 @@ int main(int argCount, const char* argv[]) ZDICT_cover_params_t coverParams = defaultCoverParams(); int cover = 1; #endif +#ifndef ZSTD_NOBENCH + BMK_advancedParams_t benchParams = BMK_initAdvancedParams(); +#endif /* init */ @@ -620,7 +622,7 @@ int main(int argCount, const char* argv[]) /* Decoding */ case 'd': #ifndef ZSTD_NOBENCH - adv.mode = BMK_decodeOnly; + benchParams.mode = BMK_decodeOnly; if (operation==zom_bench) { argument++; break; } /* benchmark decode (hidden option) */ #endif operation=zom_decompress; argument++; break; @@ -713,7 +715,7 @@ int main(int argCount, const char* argv[]) case 'p': argument++; #ifndef ZSTD_NOBENCH if ((*argument>='0') && (*argument<='9')) { - adv.additionalParam = (int)readU32FromChar(&argument); + benchParams.additionalParam = (int)readU32FromChar(&argument); } else #endif main_pause=1; @@ -826,18 +828,18 @@ int main(int argCount, const char* argv[]) /* Check if benchmark is selected */ if (operation==zom_bench) { #ifndef ZSTD_NOBENCH - adv.blockSize = blockSize; - adv.nbWorkers = nbWorkers; - adv.realTime = setRealTimePrio; - adv.nbSeconds = bench_nbSeconds; - adv.ldmFlag = ldmFlag; - adv.ldmMinMatch = g_ldmMinMatch; - adv.ldmHashLog = g_ldmHashLog; + benchParams.blockSize = blockSize; + benchParams.nbWorkers = nbWorkers; + benchParams.realTime = setRealTimePrio; + benchParams.nbSeconds = bench_nbSeconds; + benchParams.ldmFlag = ldmFlag; + benchParams.ldmMinMatch = g_ldmMinMatch; + benchParams.ldmHashLog = g_ldmHashLog; if (g_ldmBucketSizeLog != LDM_PARAM_DEFAULT) { - adv.ldmBucketSizeLog = g_ldmBucketSizeLog; + benchParams.ldmBucketSizeLog = g_ldmBucketSizeLog; } if (g_ldmHashEveryLog != LDM_PARAM_DEFAULT) { - adv.ldmHashEveryLog = g_ldmHashEveryLog; + benchParams.ldmHashEveryLog = g_ldmHashEveryLog; } if (cLevel > ZSTD_maxCLevel()) cLevel = ZSTD_maxCLevel(); @@ -852,17 +854,17 @@ int main(int argCount, const char* argv[]) int c; DISPLAYLEVEL(2, "Benchmarking %s \n", filenameTable[i]); for(c = cLevel; c <= cLevelLast; c++) { - BMK_benchFilesAdvanced(&filenameTable[i], 1, dictFileName, c, &compressionParams, g_displayLevel, &adv); + BMK_benchFilesAdvanced(&filenameTable[i], 1, dictFileName, c, &compressionParams, g_displayLevel, &benchParams); } } } else { for(; cLevel <= cLevelLast; cLevel++) { - BMK_benchFilesAdvanced(filenameTable, filenameIdx, dictFileName, cLevel, &compressionParams, g_displayLevel, &adv); + BMK_benchFilesAdvanced(filenameTable, filenameIdx, dictFileName, cLevel, &compressionParams, g_displayLevel, &benchParams); } } } else { for(; cLevel <= cLevelLast; cLevel++) { - BMK_syntheticTest(cLevel, compressibility, &compressionParams, g_displayLevel, &adv); + BMK_syntheticTest(cLevel, compressibility, &compressionParams, g_displayLevel, &benchParams); } } From ca785c4b2093dcfaccf739d769eecf93fbb01232 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Fri, 3 Aug 2018 07:59:33 -0700 Subject: [PATCH 099/372] fix .travis.yml --- .travis.yml | 19 +++---------------- 1 file changed, 3 insertions(+), 16 deletions(-) diff --git a/.travis.yml b/.travis.yml index d41a6774d..ea71818e9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,13 +10,10 @@ addons: matrix: include: # Ubuntu 14.04 -<<<<<<< HEAD - - env: Cmd='make gcc6install && CC=gcc-6 make -j all - && make clean && CC=gcc-6 make clean uasan-test-zstd' -======= - env: Cmd='make test' - - env: Cmd='make gcc6install && CC=gcc-6 make -j all && make clean && CC=gcc-6 make clean uasan-test-zstd >>>>>> 0840d02ecf74eae656e0df0d900000d9b0154cde + + - env: Cmd='make gcc6install && CC=gcc-6 make -j all + && make clean && CC=gcc-6 make clean uasan-test-zstd collect2: error: ld terminated with signal 11 [Segmentation fault], core dumped -# to be re-enabled in a few commit, as it's possible that a random code change circumvent the ld bug -# - env: Cmd='make arminstall && make aarch64fuzz' - ->>>>>>> dev - env: Cmd='make ppcinstall && make ppcfuzz' - env: Cmd='make ppcinstall && make ppc64fuzz' From 2fdab1629be5f46c778dcf9a4c0175ffff47e9ea Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Fri, 3 Aug 2018 08:30:01 -0700 Subject: [PATCH 100/372] fix unused variable warning --- programs/zstdcli.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/programs/zstdcli.c b/programs/zstdcli.c index 34f73302a..d5a2216d6 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -869,7 +869,7 @@ int main(int argCount, const char* argv[]) } #else - (void)bench_nbSeconds; (void)blockSize; (void)setRealTimePrio; (void)separateFiles; + (void)bench_nbSeconds; (void)blockSize; (void)setRealTimePrio; (void)separateFiles; (void)compressibility; #endif goto _end; } From 0feed8e1f8f1c76e3fb1b81316e31eb11decddb5 Mon Sep 17 00:00:00 2001 From: Jennifer Liu Date: Mon, 6 Aug 2018 17:54:04 -0700 Subject: [PATCH 101/372] Run non-optimize FASTCOVER 5 times in benchmark --- .../benchmarkDictBuilder/benchmark.c | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/contrib/experimental_dict_builders/benchmarkDictBuilder/benchmark.c b/contrib/experimental_dict_builders/benchmarkDictBuilder/benchmark.c index d92e8d5cb..7dade2da3 100644 --- a/contrib/experimental_dict_builders/benchmarkDictBuilder/benchmark.c +++ b/contrib/experimental_dict_builders/benchmarkDictBuilder/benchmark.c @@ -416,11 +416,13 @@ int main(int argCount, const char* argv[]) } /* for fastCover (with k and d provided) */ - const int fastResult = benchmarkDictBuilder(srcInfo, maxDictSize, NULL, NULL, NULL, &fastParam); - DISPLAYLEVEL(2, "k=%u\nd=%u\nf=%u\nsteps=%u\nsplit=%u\n", fastParam.k, fastParam.d, fastParam.f, fastParam.steps, (unsigned)(fastParam.splitPoint * 100)); - if(fastResult) { - result = 1; - goto _cleanup; + for (int i = 0; i < 5; i++) { + const int fastResult = benchmarkDictBuilder(srcInfo, maxDictSize, NULL, NULL, NULL, &fastParam); + DISPLAYLEVEL(2, "k=%u\nd=%u\nf=%u\nsteps=%u\nsplit=%u\n", fastParam.k, fastParam.d, fastParam.f, fastParam.steps, (unsigned)(fastParam.splitPoint * 100)); + if(fastResult) { + result = 1; + goto _cleanup; + } } } From 2ca7c69167e17006f867ab550408b6e0487a81e7 Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Tue, 7 Aug 2018 17:05:05 -0700 Subject: [PATCH 102/372] Fix CDict Attachment to Handle CDicts with Non-Zero Starts CDicts were previously guaranteed to be generated with `lowLimit=dictLimit=0`. This is no longer true, and so the old length and index calculations are no longer valid. This diff fixes them to handle non-zero start indices in CDicts. --- lib/compress/zstd_compress.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 3ebfd019f..ed3aab871 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -1284,8 +1284,9 @@ static size_t ZSTD_resetCCtx_usingCDict(ZSTD_CCtx* cctx, } if (attachDict) { - const U32 cdictLen = (U32)( cdict->matchState.window.nextSrc + const U32 cdictEnd = (U32)( cdict->matchState.window.nextSrc - cdict->matchState.window.base); + const U32 cdictLen = cdictEnd - cdict->matchState.window.dictLimit; if (cdictLen == 0) { /* don't even attach dictionaries with no contents */ DEBUGLOG(4, "skipping attaching empty dictionary"); @@ -1295,9 +1296,9 @@ static size_t ZSTD_resetCCtx_usingCDict(ZSTD_CCtx* cctx, /* prep working match state so dict matches never have negative indices * when they are translated to the working context's index space. */ - if (cctx->blockState.matchState.window.dictLimit < cdictLen) { + if (cctx->blockState.matchState.window.dictLimit < cdictEnd) { cctx->blockState.matchState.window.nextSrc = - cctx->blockState.matchState.window.base + cdictLen; + cctx->blockState.matchState.window.base + cdictEnd; ZSTD_window_clear(&cctx->blockState.matchState.window); } cctx->blockState.matchState.loadedDictEnd = cctx->blockState.matchState.window.dictLimit; From 93750a54a9df528c9692380b80b4ed4755f4f05a Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 8 Aug 2018 10:22:19 -0700 Subject: [PATCH 103/372] try to improve some travis test speed --- .travis.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index ea71818e9..a3b514c8e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,10 +12,11 @@ matrix: # Ubuntu 14.04 - env: Cmd='make test' - - env: Cmd='make gcc6install && CC=gcc-6 make -j all - && make clean && CC=gcc-6 make clean uasan-test-zstd Date: Wed, 8 Aug 2018 10:26:54 -0700 Subject: [PATCH 104/372] =?UTF-8?q?try=20adding=20gcc-8=C2=A0compilation?= =?UTF-8?q?=20tests?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .travis.yml | 1 + Makefile | 3 +++ 2 files changed, 4 insertions(+) diff --git a/.travis.yml b/.travis.yml index a3b514c8e..257de66e0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,6 +17,7 @@ matrix: - env: Cmd='make gcc6install libc6install && make clean && CC=gcc-6 make -j uasan-test-zstd32' - env: Cmd='make gcc7install && make clean && CC=gcc-7 make -j uasan-test-zstd' + - env: Cmd='make gcc8install && CC=gcc-8 CFLAGS=-Werror make -j all' - env: Cmd='make clang38install && CC=clang-3.8 make clean msan-test-zstd' - env: Cmd='make gcc6install && CC=gcc-6 make clean uasan-fuzztest' diff --git a/Makefile b/Makefile index 98c35a346..f67791275 100644 --- a/Makefile +++ b/Makefile @@ -287,6 +287,9 @@ gcc6install: apt-add-repo gcc7install: apt-add-repo APT_PACKAGES="libc6-dev-i386 gcc-multilib gcc-7 gcc-7-multilib" $(MAKE) apt-install +gcc8install: apt-add-repo + APT_PACKAGES="libc6-dev-i386 gcc-multilib gcc-8 gcc-8-multilib" $(MAKE) apt-install + gpp6install: apt-add-repo APT_PACKAGES="libc6-dev-i386 g++-multilib gcc-6 g++-6 g++-6-multilib" $(MAKE) apt-install From 9deef2c41bc092fa83c8c15d6bc9402b67325127 Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Wed, 8 Aug 2018 10:28:22 -0700 Subject: [PATCH 105/372] Add an Auto-Gen'ed CircleCI 2.0 Config Built via the cci-config-generator.sh script. --- .circleci/config.yml | 105 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 105 insertions(+) create mode 100644 .circleci/config.yml diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 000000000..4780c3d68 --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,105 @@ +# This configuration was automatically generated from a CircleCI 1.0 config. +# It should include any build commands you had along with commands that CircleCI +# inferred from your project structure. We strongly recommend you read all the +# comments in this file to understand the structure of CircleCI 2.0, as the idiom +# for configuration has changed substantially in 2.0 to allow arbitrary jobs rather +# than the prescribed lifecycle of 1.0. In general, we recommend using this generated +# configuration as a reference rather than using it in production, though in most +# cases it should duplicate the execution of your original 1.0 config. +version: 2 +jobs: + build: + working_directory: ~/felixhandte/zstd + parallelism: 1 + shell: /bin/bash --login + # CircleCI 2.0 does not support environment variables that refer to each other the same way as 1.0 did. + # If any of these refer to each other, rewrite them so that they don't or see https://circleci.com/docs/2.0/env-vars/#interpolating-environment-variables-to-set-other-environment-variables . + environment: + CIRCLE_ARTIFACTS: /tmp/circleci-artifacts + CIRCLE_TEST_REPORTS: /tmp/circleci-test-results + # In CircleCI 1.0 we used a pre-configured image with a large number of languages and other packages. + # In CircleCI 2.0 you can now specify your own image, or use one of our pre-configured images. + # The following configuration line tells CircleCI to use the specified docker image as the runtime environment for you job. + # We have selected a pre-built image that mirrors the build environment we use on + # the 1.0 platform, but we recommend you choose an image more tailored to the needs + # of each job. For more information on choosing an image (or alternatively using a + # VM instead of a container) see https://circleci.com/docs/2.0/executor-types/ + # To see the list of pre-built images that CircleCI provides for most common languages see + # https://circleci.com/docs/2.0/circleci-images/ + docker: + - image: circleci/build-image:ubuntu-14.04-XXL-upstart-1189-5614f37 + command: /sbin/init + steps: + # Machine Setup + # If you break your build into multiple jobs with workflows, you will probably want to do the parts of this that are relevant in each + # The following `checkout` command checks out your code to your working directory. In 1.0 we did this implicitly. In 2.0 you can choose where in the course of a job your code should be checked out. + - checkout + # Prepare for artifact and test results collection equivalent to how it was done on 1.0. + # In many cases you can simplify this from what is generated here. + # 'See docs on artifact collection here https://circleci.com/docs/2.0/artifacts/' + - run: mkdir -p $CIRCLE_ARTIFACTS $CIRCLE_TEST_REPORTS + # Dependencies + # This would typically go in either a build or a build-and-test job when using workflows + # Restore the dependency cache + - restore_cache: + keys: + # This branch if available + - v1-dep-{{ .Branch }}- + # Default branch if not + - v1-dep-dev- + # Any branch if there are none on the default branch - this should be unnecessary if you have your default branch configured correctly + - v1-dep- + # This is based on your 1.0 configuration file or project settings + - run: sudo dpkg --add-architecture i386 + - run: sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test; sudo apt-get -y -qq update + - run: sudo apt-get -y install gcc-powerpc-linux-gnu gcc-arm-linux-gnueabi libc6-dev-armel-cross gcc-aarch64-linux-gnu libc6-dev-arm64-cross + # Save dependency cache + - save_cache: + key: v1-dep-{{ .Branch }}-{{ epoch }} + paths: + # This is a broad list of cache paths to include many possible development environments + # You can probably delete some of these entries + - vendor/bundle + - ~/virtualenvs + - ~/.m2 + - ~/.ivy2 + - ~/.bundle + - ~/.go_workspace + - ~/.gradle + - ~/.cache/bower + # Test + # This would typically be a build job when using workflows, possibly combined with build + # This is based on your 1.0 configuration file or project settings + - run: | + if [[ "$CIRCLE_NODE_INDEX" == "0" ]] ; then cc -v; CFLAGS="-O0 -Werror" make all && make clean; fi && + if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make gnu90build && make clean; fi + - run: | + if [[ "$CIRCLE_NODE_INDEX" == "0" ]] ; then make c99build && make clean; fi && + if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make gnu99build && make clean; fi + - run: | + if [[ "$CIRCLE_NODE_INDEX" == "0" ]] ; then make c11build && make clean; fi && + if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make ppc64build && make clean; fi + - run: | + if [[ "$CIRCLE_NODE_INDEX" == "0" ]] ; then make aarch64build && make clean; fi && + if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make ppcbuild && make clean; fi + - run: | + if [[ "$CIRCLE_NODE_INDEX" == "0" ]] ; then make -j regressiontest && make clean; fi && + if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make armbuild && make clean; fi + - run: | + if [[ "$CIRCLE_NODE_INDEX" == "0" ]] ; then make shortest && make clean; fi && + if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make -C tests test-legacy test-longmatch test-symbols && make clean; fi + - run: | + if [[ "$CIRCLE_NODE_INDEX" == "0" ]] ; then make cxxtest && make clean; fi && + if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make -C lib libzstd-nomt && make clean; fi + # This is based on your 1.0 configuration file or project settings + - run: echo Circle CI tests finished + # Teardown + # If you break your build into multiple jobs with workflows, you will probably want to do the parts of this that are relevant in each + # Save test results + - store_test_results: + path: /tmp/circleci-test-results + # Save artifacts + - store_artifacts: + path: /tmp/circleci-artifacts + - store_artifacts: + path: /tmp/circleci-test-results From 4f5e4fe9ed27dd7cee7cf46cebf2b15e8014378b Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Wed, 8 Aug 2018 12:32:55 -0700 Subject: [PATCH 106/372] Fix Path --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 4780c3d68..389fb11ae 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -9,7 +9,7 @@ version: 2 jobs: build: - working_directory: ~/felixhandte/zstd + working_directory: ~/facebook/zstd parallelism: 1 shell: /bin/bash --login # CircleCI 2.0 does not support environment variables that refer to each other the same way as 1.0 did. From 0ede1735b015ed993ece4a43778334a1a72ba78a Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Wed, 8 Aug 2018 12:31:37 -0700 Subject: [PATCH 107/372] Remove Old CircleCI Config --- circle.yml | 63 ------------------------------------------------------ 1 file changed, 63 deletions(-) delete mode 100644 circle.yml diff --git a/circle.yml b/circle.yml deleted file mode 100644 index ed50d59e5..000000000 --- a/circle.yml +++ /dev/null @@ -1,63 +0,0 @@ -dependencies: - override: - - sudo dpkg --add-architecture i386 - - sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test; sudo apt-get -y -qq update - - sudo apt-get -y install gcc-powerpc-linux-gnu gcc-arm-linux-gnueabi libc6-dev-armel-cross gcc-aarch64-linux-gnu libc6-dev-arm64-cross - -test: - override: - - ? | - if [[ "$CIRCLE_NODE_INDEX" == "0" ]] ; then cc -v; CFLAGS="-O0 -Werror" make all && make clean; fi && - if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make gnu90build && make clean; fi - : - parallel: true - - ? | - if [[ "$CIRCLE_NODE_INDEX" == "0" ]] ; then make c99build && make clean; fi && - if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make gnu99build && make clean; fi - : - parallel: true - - ? | - if [[ "$CIRCLE_NODE_INDEX" == "0" ]] ; then make c11build && make clean; fi && - if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make ppc64build && make clean; fi - : - parallel: true - - ? | - if [[ "$CIRCLE_NODE_INDEX" == "0" ]] ; then make aarch64build && make clean; fi && - if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make ppcbuild && make clean; fi - : - parallel: true - - ? | - if [[ "$CIRCLE_NODE_INDEX" == "0" ]] ; then make -j regressiontest && make clean; fi && - if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make armbuild && make clean; fi - : - parallel: true - - ? | - if [[ "$CIRCLE_NODE_INDEX" == "0" ]] ; then make shortest && make clean; fi && - if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make -C tests test-legacy test-longmatch test-symbols && make clean; fi - : - parallel: true - - ? | - if [[ "$CIRCLE_NODE_INDEX" == "0" ]] ; then make cxxtest && make clean; fi && - if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make -C lib libzstd-nomt && make clean; fi - : - parallel: true - - post: - - echo Circle CI tests finished - - # Longer tests - #- make -C tests test-zstd-nolegacy && make clean - #- pyenv global 3.4.4; make -C tests versionsTest && make clean - #- make zlibwrapper && make clean - #- gcc -v; make -C tests test32 MOREFLAGS="-I/usr/include/x86_64-linux-gnu" && make clean - #- make uasan && make clean - #- make asan32 && make clean - #- make -C tests test32 CC=clang MOREFLAGS="-g -fsanitize=address -I/usr/include/x86_64-linux-gnu" - # Valgrind tests - #- CFLAGS="-O1 -g" make -C zlibWrapper valgrindTest && make clean - #- make -C tests valgrindTest && make clean - # ARM, AArch64, PowerPC, PowerPC64 tests - #- make ppctest && make clean - #- make ppc64test && make clean - #- make armtest && make clean - #- make aarch64test && make clean From d1afd48e44404200e3ec1ee003b0e9bfcc8d7ee8 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 8 Aug 2018 12:37:08 -0700 Subject: [PATCH 108/372] some errors may only happen when optimizations are enabled --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 257de66e0..36df67052 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,7 +17,7 @@ matrix: - env: Cmd='make gcc6install libc6install && make clean && CC=gcc-6 make -j uasan-test-zstd32' - env: Cmd='make gcc7install && make clean && CC=gcc-7 make -j uasan-test-zstd' - - env: Cmd='make gcc8install && CC=gcc-8 CFLAGS=-Werror make -j all' + - env: Cmd='make gcc8install && CC=gcc-8 CFLAGS="-Werror -O3" make -j all' - env: Cmd='make clang38install && CC=clang-3.8 make clean msan-test-zstd' - env: Cmd='make gcc6install && CC=gcc-6 make clean uasan-fuzztest' From 6518745024bf9e3d804d998c14f41144e4c16d91 Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Wed, 8 Aug 2018 14:09:12 -0700 Subject: [PATCH 109/372] Preserve Commented-Out Longer Tests --- .circleci/config.yml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 389fb11ae..7af8b41ed 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -103,3 +103,20 @@ jobs: path: /tmp/circleci-artifacts - store_artifacts: path: /tmp/circleci-test-results + + # Longer tests + #- make -C tests test-zstd-nolegacy && make clean + #- pyenv global 3.4.4; make -C tests versionsTest && make clean + #- make zlibwrapper && make clean + #- gcc -v; make -C tests test32 MOREFLAGS="-I/usr/include/x86_64-linux-gnu" && make clean + #- make uasan && make clean + #- make asan32 && make clean + #- make -C tests test32 CC=clang MOREFLAGS="-g -fsanitize=address -I/usr/include/x86_64-linux-gnu" + # Valgrind tests + #- CFLAGS="-O1 -g" make -C zlibWrapper valgrindTest && make clean + #- make -C tests valgrindTest && make clean + # ARM, AArch64, PowerPC, PowerPC64 tests + #- make ppctest && make clean + #- make ppc64test && make clean + #- make armtest && make clean + #- make aarch64test && make clean From cffb6da339970c7a5334e52abdf96493764561f7 Mon Sep 17 00:00:00 2001 From: George Lu Date: Wed, 6 Jun 2018 16:19:09 -0700 Subject: [PATCH 110/372] Parses additional parameters Additional constraint checking Minor fixes more param parsing Add Memory Change paramVariation work on feasibility reformat bench Changed Paramgrill to use bench.c benchmarking customlevel macro Printing Flag Minor changes Explicit casting Makefile fix casting, type fix Printing Flag Minor Changes comments, helper fn's --- programs/bench.c | 2 + programs/bench.h | 1 + tests/paramgrill.c | 369 ++++++++++++++++++++++++++++++++++++++------- 3 files changed, 320 insertions(+), 52 deletions(-) diff --git a/programs/bench.c b/programs/bench.c index 76d1ff6dd..2fa9262c9 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -692,6 +692,7 @@ static BMK_return_t BMK_benchMemAdvancedNoAlloc( } DISPLAYLEVEL(2, "%2i#\n", cLevel); } /* Bench */ + results.result.cMem = ZSTD_sizeof_CCtx(ctx); return results; } @@ -731,6 +732,7 @@ BMK_return_t BMK_benchMemAdvanced(const void* srcBuffer, size_t srcSize, srcBuffer, srcSize, fileSizes, nbFiles, cLevel, comprParams, dictBuffer, dictBufferSize, ctx, dctx, displayLevel, displayName, adv); } + /* clean up */ BMK_freeTimeState(timeStateCompress); BMK_freeTimeState(timeStateDecompress); diff --git a/programs/bench.h b/programs/bench.h index 87cf56380..f1ac255f8 100644 --- a/programs/bench.h +++ b/programs/bench.h @@ -34,6 +34,7 @@ typedef struct { size_t cSize; double cSpeed; /* bytes / sec */ double dSpeed; + size_t cMem; } BMK_result_t; ERROR_STRUCT(BMK_result_t, BMK_return_t); diff --git a/tests/paramgrill.c b/tests/paramgrill.c index 025bc6aad..81c6dffc2 100644 --- a/tests/paramgrill.c +++ b/tests/paramgrill.c @@ -63,6 +63,16 @@ static const int g_maxNbVariations = 64; #define MAX(a,b) ( (a) > (b) ? (a) : (b) ) #define CUSTOM_LEVEL 99 +/* indices for each of the variables */ +#define WLOG_IND 0 +#define CLOG_IND 1 +#define HLOG_IND 2 +#define SLOG_IND 3 +#define SLEN_IND 4 +#define TLEN_IND 5 +#define STRT_IND 6 +#define NUM_PARAMS 7 + /*-************************************ * Benchmark Parameters **************************************/ @@ -140,6 +150,13 @@ static unsigned longCommandWArg(const char** stringPtr, const char* longCommand) return result; } + +typedef struct { + U32 cSpeed; /* bytes / sec */ + U32 dSpeed; + U32 Mem; /* bytes */ +} constraint_t; + /*-******************************************************* * Bench functions *********************************************************/ @@ -340,7 +357,6 @@ static int BMK_seed(winnerInfo_t* winners, const ZSTD_compressionParameters para return better; } - /* nullified useless params, to ensure count stats */ static ZSTD_compressionParameters* sanitizeParams(ZSTD_compressionParameters params) { @@ -354,8 +370,114 @@ static ZSTD_compressionParameters* sanitizeParams(ZSTD_compressionParameters par return &g_params; } +/* res should be NUM_PARAMS size */ +static int variableParams(const ZSTD_compressionParameters paramConstraints, U32* res) { + int j = 0; + if(!paramConstraints.windowLog) { + res[j] = WLOG_IND; + j++; + } + if(!paramConstraints.chainLog) { + res[j] = CLOG_IND; + j++; + } + if(!paramConstraints.hashLog) { + res[j] = HLOG_IND; + j++; + } + if(!paramConstraints.searchLog) { + res[j] = SLOG_IND; + j++; + } + if(!paramConstraints.searchLength) { + res[j] = SLEN_IND; + j++; + } + if(!paramConstraints.targetLength) { + res[j] = TLEN_IND; + j++; + } + if(!(U32)paramConstraints.strategy) { + res[j] = STRT_IND; + j++; + } + return j; +} -static void paramVariation(ZSTD_compressionParameters* ptr) +/* computes inverse of above array, returns same number, -1 = unused ind */ +static int inverseVariableParams(const ZSTD_compressionParameters paramConstraints, U32* res) { + int j = 0; + if(!paramConstraints.windowLog) { + res[WLOG_IND] = j; + j++; + } else { + res[WLOG_IND] = -1; + } + if(!paramConstraints.chainLog) { + res[j] = CLOG_IND; + j++; + } else { + res[WLOG_IND] = -1; + } + if(!paramConstraints.hashLog) { + res[j] = HLOG_IND; + j++; + } else { + res[WLOG_IND] = -1; + } + if(!paramConstraints.searchLog) { + res[j] = SLOG_IND; + j++; + } else { + res[WLOG_IND] = -1; + } + if(!paramConstraints.searchLength) { + res[j] = SLEN_IND; + j++; + } else { + res[WLOG_IND] = -1; + } + if(!paramConstraints.targetLength) { + res[j] = TLEN_IND; + j++; + } else { + res[WLOG_IND] = -1; + } + if(!(U32)paramConstraints.strategy) { + res[j] = STRT_IND; + j++; + } else { + res[WLOG_IND] = -1; + } + return j; +} + +/* amt will probably always be \pm 1? */ +/* slight change from old paramVariation, targetLength can only take on powers of 2 now (999 ~= 1024?) */ +static void paramVaryOnce(U32 paramIndex, int amt, ZSTD_compressionParameters* ptr) { + switch(paramIndex) + { + case WLOG_IND: ptr->windowLog += amt; break; + case CLOG_IND: ptr->chainLog += amt; break; + case HLOG_IND: ptr->hashLog += amt; break; + case SLOG_IND: ptr->searchLog += amt; break; + case SLEN_IND: ptr->searchLength += amt; break; + case TLEN_IND: + if(amt > 0) { + ptr->targetLength <<= amt; + } else { + ptr->targetLength >>= -amt; + } + break; + case STRT_IND: ptr->strategy += amt; break; + default: break; + } +} + +//Don't fuzz fixed variables. +//turn pcs to pcs array with macro for params. +//pass in variation array from variableParams +static void paramVariation(ZSTD_compressionParameters* ptr, const U32* varyParams, const int varyLen) { ZSTD_compressionParameters p; U32 validated = 0; @@ -363,40 +485,13 @@ static void paramVariation(ZSTD_compressionParameters* ptr) U32 nbChanges = (FUZ_rand(&g_rand) & 3) + 1; p = *ptr; for ( ; nbChanges ; nbChanges--) { - const U32 changeID = FUZ_rand(&g_rand) % 14; - switch(changeID) - { - case 0: - p.chainLog++; break; - case 1: - p.chainLog--; break; - case 2: - p.hashLog++; break; - case 3: - p.hashLog--; break; - case 4: - p.searchLog++; break; - case 5: - p.searchLog--; break; - case 6: - p.windowLog++; break; - case 7: - p.windowLog--; break; - case 8: - p.searchLength++; break; - case 9: - p.searchLength--; break; - case 10: - p.strategy = (ZSTD_strategy)(((U32)p.strategy)+1); break; - case 11: - p.strategy = (ZSTD_strategy)(((U32)p.strategy)-1); break; - case 12: - p.targetLength *= 1 + ((double)(FUZ_rand(&g_rand)&255)) / 256.; break; - case 13: - p.targetLength /= 1 + ((double)(FUZ_rand(&g_rand)&255)) / 256.; break; - } + const U32 changeID = FUZ_rand(&g_rand) % (2 * varyLen); + paramVaryOnce(varyParams[changeID >> 1], ((changeID & 1) << 1) - 1, &p); } validated = !ZSTD_isError(ZSTD_checkCParams(p)); + + //Make sure memory is at least close to feasible? + //ZSTD_estimateCCtxSize thing. } *ptr = p; } @@ -418,12 +513,14 @@ static void playAround(FILE* f, winnerInfo_t* winners, { int nbVariations = 0; UTIL_time_t const clockStart = UTIL_getTime(); + const U32 unconstrained[NUM_PARAMS] = { 0, 1, 2, 3, 4, 5, 6 }; + while (UTIL_clockSpanMicro(clockStart) < g_maxVariationTime) { ZSTD_compressionParameters p = params; if (nbVariations++ > g_maxNbVariations) break; - paramVariation(&p); + paramVariation(&p, unconstrained, 7); /* exclude faster if already played params */ if (FUZ_rand(&g_rand) & ((1 << NB_TESTS_PLAYED(p))-1)) @@ -459,6 +556,24 @@ static ZSTD_compressionParameters randomParams(void) return p; } +static ZSTD_compressionParameters randomConstrainedParams(ZSTD_compressionParameters pc) +{ + ZSTD_compressionParameters p; + U32 validated = 0; + while (!validated) { + /* totally random entry */ + if(!pc.chainLog) p.chainLog = (FUZ_rand(&g_rand) % (ZSTD_CHAINLOG_MAX+1 - ZSTD_CHAINLOG_MIN)) + ZSTD_CHAINLOG_MIN; + if(!pc.chainLog) p.hashLog = (FUZ_rand(&g_rand) % (ZSTD_HASHLOG_MAX+1 - ZSTD_HASHLOG_MIN)) + ZSTD_HASHLOG_MIN; + if(!pc.chainLog) p.searchLog = (FUZ_rand(&g_rand) % (ZSTD_SEARCHLOG_MAX+1 - ZSTD_SEARCHLOG_MIN)) + ZSTD_SEARCHLOG_MIN; + if(!pc.chainLog) p.windowLog = (FUZ_rand(&g_rand) % (ZSTD_WINDOWLOG_MAX+1 - ZSTD_WINDOWLOG_MIN)) + ZSTD_WINDOWLOG_MIN; + if(!pc.chainLog) p.searchLength=(FUZ_rand(&g_rand) % (ZSTD_SEARCHLENGTH_MAX+1 - ZSTD_SEARCHLENGTH_MIN)) + ZSTD_SEARCHLENGTH_MIN; + if(!pc.chainLog) p.targetLength=(FUZ_rand(&g_rand) % (512)) + 1; //ZSTD_TARGETLENGTH_MIN; //change to 2^[0,10?] + if(!pc.chainLog) p.strategy = (ZSTD_strategy) (FUZ_rand(&g_rand) % (ZSTD_btultra +1)); + validated = !ZSTD_isError(ZSTD_checkCParams(p)); + } + return p; +} + static void BMK_selectRandomStart( FILE* f, winnerInfo_t* winners, const void* srcBuffer, size_t srcSize, @@ -638,14 +753,86 @@ static void BMK_translateAdvancedParams(ZSTD_compressionParameters params) params.windowLog, params.chainLog, params.hashLog, params.searchLog, params.searchLength, params.targetLength, (U32)(params.strategy)); } +//Results currently don't capture memory usage or anything. +//parameter feasibility is not checked, should just be restricted from use. +static int feasible(BMK_result_t results, constraint_t target) { + return (results.cSpeed >= target.cSpeed) && (results.dSpeed >= target.dSpeed) && (results.cMem <= target.Mem || !target.Mem); +} + +/* returns 1 if result2 is strictly 'better' than result1 */ +static int objective_lt(BMK_result_t result1, BMK_result_t result2) { + return (result1.cSize > result2.cSize) || (result1.cSize == result2.cSize && result2.cSpeed > result1.cSpeed); +} + +/* res gives array dimensions, should be size NUM_PARAMS */ +static size_t computeStateSize(const ZSTD_compressionParameters paramConstraints, U32* res) { + int ind = 0; + size_t base = 1; + if(!paramConstraints.windowLog) { res[ind] = ZSTD_WINDOWLOG_MAX - ZSTD_WINDOWLOG_MIN + 1; base *= res[ind]; ind++; } + if(!paramConstraints.chainLog) { res[ind] = ZSTD_CHAINLOG_MAX - ZSTD_CHAINLOG_MIN + 1; base *= res[ind]; ind++; } + if(!paramConstraints.hashLog) { res[ind] = ZSTD_HASHLOG_MAX - ZSTD_HASHLOG_MIN + 1; base *= res[ind]; ind++; } + if(!paramConstraints.searchLog) { res[ind] = ZSTD_SEARCHLOG_MAX - ZSTD_SEARCHLOG_MIN + 1; base *= res[ind]; ind++; } + if(!paramConstraints.searchLength) { res[ind] = ZSTD_SEARCHLENGTH_MAX - ZSTD_SEARCHLENGTH_MIN + 1; base *= res[ind]; ind++; } + if(!paramConstraints.targetLength) { res[ind] = 11; base *= res[ind]; ind++; } //restricting from 2^[0,10], no such macros + if(!(U32)paramConstraints.strategy) { res[ind] = 8; base *= 8; } //not strictly true, maybe would want to case on this. + + return base; +} + + +static unsigned calcViolation(BMK_result_t results, constraint_t target) { + int diffcSpeed = MAX(target.cSpeed - results.cSpeed, 0); + int diffdSpeed = MAX(target.dSpeed - results.dSpeed, 0); + int diffcMem = MAX(results.cMem - target.Mem, 0); + return diffcSpeed + diffdSpeed + diffcMem; +} + +/* finds some set of parameters which fulfills req's + * Prioritize highest / try to locally minimize sum? + * Is it ever useful to go out of the param constraints? ? + * random / perturb when revisit? + * momentum? + */ +static ZSTD_compressionParameters findFeasible(constraint_t target, ZSTD_compressionParameters paramTarget) { + unsigned violation; + + ZSTD_compressionParameters winner = randomConstrainedParams(paramTarget); + //just use g_alreadyTested and xxhash? + BYTE* memotable; + do { + //prioritize memory + if(diffcMem >= diffcSpeed && diffcMem >= diffdSpeed) { + + //prioritize compression Speed + } else if (diffcSpeeed >= diffdSpeed && diffcSpeed >= diffcMem) { + + //prioritize decompressionSpeed + } else { + + } + violation = calcViolation(result, target); + } while(objective); + if(validate) { + DISPLAY("Feasible Point Found\n"); + return winner; + } else { + DISPLAY("No solution found\n"); + ZSTD_compressionParameters ret = { 0, 0, 0, 0, 0, 0, 0 }; + return ret; + } +} + /* optimizeForSize(): - * targetSpeed : expressed in MB/s */ -int optimizeForSize(const char* inFileName, U32 targetSpeed) + * targetSpeed : expressed in B/s */ +/* if state space is small (from paramTarget) */ +int optimizeForSize(const char* inFileName, constraint_t target, ZSTD_compressionParameters paramTarget) { FILE* const inFile = fopen( inFileName, "rb" ); U64 const inFileSize = UTIL_getFileSize(inFileName); size_t benchedSize = BMK_findMaxMem(inFileSize*3) / 3; void* origBuff; + U32 paramVarArray [NUM_PARAMS]; + int paramCount = variableParams(paramTarget, paramVarArray); /* Init */ if (inFile==NULL) { DISPLAY( "Pb opening %s\n", inFileName); return 11; } if (inFileSize == UTIL_FILESIZE_UNKNOWN) { @@ -682,8 +869,11 @@ int optimizeForSize(const char* inFileName, U32 targetSpeed) /* bench */ DISPLAY("\r%79s\r", ""); - DISPLAY("optimizing for %s - limit speed %u MB/s \n", inFileName, targetSpeed); - targetSpeed *= 1000000; + DISPLAY("optimizing for %s", inFileName); + if(target.cSpeed != 0) { DISPLAY(" - limit compression speed %u MB/s", target.cSpeed / 1000000); } + if(target.dSpeed != 0) { DISPLAY(" - limit decompression speed %u MB/s", target.dSpeed / 1000000); } + if(target.Mem != 0) { DISPLAY(" - limit memory %u MB", target.Mem / 1000000); } + DISPLAY("\n"); { ZSTD_CCtx* const ctx = ZSTD_createCCtx(); ZSTD_DCtx* const dctx = ZSTD_createDCtx(); winnerInfo_t winner; @@ -696,23 +886,22 @@ int optimizeForSize(const char* inFileName, U32 targetSpeed) winner.result.cSize = (size_t)(-1); /* find best solution from default params */ + //Can't do this iteration normally w/ cparameter constraints { const int maxSeeds = g_noSeed ? 1 : ZSTD_maxCLevel(); int i; for (i=1; i<=maxSeeds; i++) { ZSTD_compressionParameters const CParams = ZSTD_getCParams(i, blockSize, 0); BMK_benchParam(&candidate, origBuff, benchedSize, ctx, dctx, CParams); - if (candidate.cSpeed < (double)targetSpeed) { + if (!feasible(candidate, target) ) { break; } - if ( (candidate.cSize < winner.result.cSize) - | ((candidate.cSize == winner.result.cSize) & (candidate.cSpeed > winner.result.cSpeed)) ) + if (feasible(candidate,target) && objective_lt(winner.result, candidate)) { winner.params = CParams; winner.result = candidate; BMK_printWinner(stdout, i, winner.result, winner.params, benchedSize); } } } - BMK_printWinner(stdout, CUSTOM_LEVEL, winner.result, winner.params, benchedSize); BMK_translateAdvancedParams(winner.params); @@ -721,7 +910,7 @@ int optimizeForSize(const char* inFileName, U32 targetSpeed) { time_t const grillStart = time(NULL); do { ZSTD_compressionParameters params = winner.params; - paramVariation(¶ms); + paramVariation(¶ms, paramVarArray, paramCount); if ((FUZ_rand(&g_rand) & 31) == 3) params = randomParams(); /* totally random config to improve search space */ params = ZSTD_adjustCParams(params, blockSize, 0); @@ -733,9 +922,7 @@ int optimizeForSize(const char* inFileName, U32 targetSpeed) BMK_benchParam(&candidate, origBuff, benchedSize, ctx, dctx, params); /* improvement found => new winner */ - if ( (candidate.cSpeed > targetSpeed) - & ( (candidate.cSize < winner.result.cSize) - | ((candidate.cSize == winner.result.cSize) & (candidate.cSpeed > winner.result.cSpeed)) ) ) + if (feasible(candidate,target) && objective_lt(winner.result, candidate)) { winner.params = params; winner.result = candidate; @@ -744,8 +931,13 @@ int optimizeForSize(const char* inFileName, U32 targetSpeed) } } while (BMK_timeSpan(grillStart) < g_grillDuration_s); } - /* end summary */ + /* no solution found */ + if(winner.result.cSize == (size_t)-1) { + DISPLAY("No feasible solution found\n"); + return 1; + } + /* end summary */ BMK_printWinner(stdout, CUSTOM_LEVEL, winner.result, winner.params, benchedSize); BMK_translateAdvancedParams(winner.params); DISPLAY("grillParams size - optimizer completed \n"); @@ -834,7 +1026,9 @@ int main(int argc, const char** argv) const char* input_filename=0; U32 optimizer = 0; U32 main_pause = 0; - U32 targetSpeed = 0; + + constraint_t target = { 0 , 0, 0 }; //0 for anything unset + ZSTD_compressionParameters paramTarget = { 0, 0, 0, 0, 0, 0, 0 }; assert(argc>=1); /* for exename */ @@ -847,8 +1041,31 @@ int main(int argc, const char** argv) if(!strcmp(argument,"--no-seed")) { g_noSeed = 1; continue; } + if (longCommandWArg(&argument, "--optimize=")) { + optimizer = 1; + for ( ; ;) { + if (longCommandWArg(&argument, "windowLog=") || longCommandWArg(&argument, "wlog=")) { paramTarget.windowLog = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; } + if (longCommandWArg(&argument, "chainLog=") || longCommandWArg(&argument, "clog=")) { paramTarget.chainLog = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; } + if (longCommandWArg(&argument, "hashLog=") || longCommandWArg(&argument, "hlog=")) { paramTarget.hashLog = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; } + if (longCommandWArg(&argument, "searchLog=") || longCommandWArg(&argument, "slog=")) { paramTarget.searchLog = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; } + if (longCommandWArg(&argument, "searchLength=") || longCommandWArg(&argument, "slen=")) { paramTarget.searchLength = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; } + if (longCommandWArg(&argument, "targetLength=") || longCommandWArg(&argument, "tlen=")) { paramTarget.targetLength = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; } + if (longCommandWArg(&argument, "strategy=") || longCommandWArg(&argument, "strat=")) { paramTarget.strategy = (ZSTD_strategy)(readU32FromChar(&argument)); if (argument[0]==',') { argument++; continue; } else break; } + if (longCommandWArg(&argument, "compressionSpeed=") || longCommandWArg(&argument, "cSpeed=")) { target.cSpeed = readU32FromChar(&argument) * 1000000; if (argument[0]==',') { argument++; continue; } else break; } + if (longCommandWArg(&argument, "decompressionSpeed=") || longCommandWArg(&argument, "dSpeed=")) { target.dSpeed = readU32FromChar(&argument) * 1000000; if (argument[0]==',') { argument++; continue; } else break; } + if (longCommandWArg(&argument, "compressionMemory=") || longCommandWArg(&argument, "cMem=")) { target.Mem = readU32FromChar(&argument) * 1000000; if (argument[0]==',') { argument++; continue; } else break; } + /* in MB or MB/s */ + DISPLAY("invalid optimization parameter \n"); + return 1; + } + + if (argument[0] != 0) { + DISPLAY("invalid --optimize= format\n"); + return 1; /* check the end of string */ + } + continue; + } else if (longCommandWArg(&argument, "--zstd=")) { /* Decode command (note : aggregated commands are allowed) */ - if (longCommandWArg(&argument, "--zstd=")) { g_singleRun = 1; g_params = ZSTD_getCParams(2, g_blockSize, 0); for ( ; ;) { @@ -868,6 +1085,7 @@ int main(int argc, const char** argv) DISPLAY("invalid --zstd= format\n"); return 1; /* check the end of string */ } + continue; /* if not return, success */ } else if (argument[0]=='-') { argument++; @@ -900,7 +1118,54 @@ int main(int argc, const char** argv) case 'O': argument++; optimizer = 1; - targetSpeed = readU32FromChar(&argument); + for ( ; ; ) { + switch(*argument) + { + /* Inputs in MB or MB/s */ + case 'C': + argument++; + target.cSpeed = readU32FromChar(&argument) * 1000000; + continue; + case 'D': + argument++; + target.dSpeed = readU32FromChar(&argument) * 1000000; + continue; + case 'M': + argument++; + target.Mem = readU32FromChar(&argument) * 1000000; + continue; + case 'w': + argument++; + paramTarget.windowLog = readU32FromChar(&argument); + continue; + case 'c': + argument++; + paramTarget.chainLog = readU32FromChar(&argument); + continue; + case 'h': + argument++; + paramTarget.hashLog = readU32FromChar(&argument); + continue; + case 's': + argument++; + paramTarget.searchLog = readU32FromChar(&argument); + continue; + case 'l': /* search length */ + argument++; + paramTarget.searchLength = readU32FromChar(&argument); + continue; + case 't': /* target length */ + argument++; + paramTarget.targetLength = readU32FromChar(&argument); + continue; + case 'S': /* strategy */ + argument++; + paramTarget.strategy = (ZSTD_strategy)readU32FromChar(&argument); + continue; + default : ; + } + break; + } break; /* Run Single conf */ @@ -990,7 +1255,7 @@ int main(int argc, const char** argv) } } else { if (optimizer) { - result = optimizeForSize(input_filename, targetSpeed); + result = optimizeForSize(input_filename, target, paramTarget); } else { result = benchFiles(argv+filenamesStart, argc-filenamesStart); } } From 5f49034520940af38deb5e13d9d624afc7ffe534 Mon Sep 17 00:00:00 2001 From: George Lu Date: Mon, 18 Jun 2018 11:59:45 -0700 Subject: [PATCH 111/372] Working V1 --- programs/bench.c | 95 ++-- programs/bench.h | 3 + tests/Makefile | 2 +- tests/paramgrill.c | 1256 ++++++++++++++++++++++++++++++++++++++++---- 4 files changed, 1213 insertions(+), 143 deletions(-) diff --git a/programs/bench.c b/programs/bench.c index 2fa9262c9..0d35f3eb3 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -85,7 +85,7 @@ static UTIL_time_t g_displayClock = UTIL_TIME_INITIALIZER; * Exceptions ***************************************/ #ifndef DEBUG -# define DEBUG 0 +# define DEBUG 1 #endif #define DEBUGOUTPUT(...) { if (DEBUG) DISPLAY(__VA_ARGS__); } @@ -281,12 +281,14 @@ static size_t local_defaultDecompress( } +volatile char g_touched; + /* initFn will be measured once, bench fn will be measured x times */ /* benchFn should return error value or out Size */ /* takes # of blocks and list of size & stuff for each. */ /* only does looping */ /* note time per loop could be zero if interval too short */ -BMK_customReturn_t BMK_benchFunction( +BMK_customReturn_t __attribute__((optimize("O0"))) BMK_benchFunction( BMK_benchFn_t benchFn, void* benchPayload, BMK_initFn_t initFn, void* initPayload, size_t blockCount, @@ -299,16 +301,6 @@ BMK_customReturn_t BMK_benchFunction( BMK_customReturn_t retval; UTIL_time_t clockStart; - { - unsigned i; - for(i = 0; i < blockCount; i++) { - memset(dstBlockBuffers[i], 0xE5, dstBlockCapacities[i]); /* warm up and erase result buffer */ - } - - UTIL_sleepMilli(5); /* give processor time to other processes */ - UTIL_waitForNextTick(); - } - if(!nbLoops) { EXM_THROW_ND(1, BMK_customReturn_t, "nbLoops must be nonzero \n"); } @@ -317,6 +309,22 @@ BMK_customReturn_t BMK_benchFunction( srcSize += srcBlockSizes[ind]; } + { + unsigned i, j; + for(i = 0; i < blockCount; i++) { + for(j = 0; j < srcBlockSizes[i]; j++) { + g_touched = ((const char*)srcBlockBuffers[i])[j]; /* touch */ + } + } + for(i = 0; i < blockCount; i++) { + memset(dstBlockBuffers[i], 0xE5, dstBlockCapacities[i]); /* warm up and erase result buffer */ + //this is written at end proc? where did compressed data get overwritten ny this? + } + + UTIL_sleepMilli(5); /* give processor time to other processes */ + UTIL_waitForNextTick(); + } + { unsigned i, j, firstIter = 1; clockStart = UTIL_getTime(); @@ -327,9 +335,9 @@ BMK_customReturn_t BMK_benchFunction( if(ZSTD_isError(res)) { EXM_THROW_ND(2, BMK_customReturn_t, "Function benchmarking failed on block %u of size %u : %s \n", j, (U32)dstBlockCapacities[j], ZSTD_getErrorName(res)); - } else if(firstIter) { + } else if(firstIter) { dstSize += res; - } + } } firstIter = 0; } @@ -393,7 +401,7 @@ BMK_customTimedReturn_t BMK_benchFunctionTimed( { U64 const loopDuration = r.result.result.nanoSecPerRun * cont->nbLoops; r.completed = (cont->timeRemaining <= loopDuration); cont->timeRemaining -= loopDuration; - if (loopDuration > 0) { + if (loopDuration > (TIMELOOP_NANOSEC / 100)) { fastest = MIN(fastest, r.result.result.nanoSecPerRun); if(loopDuration >= MINUSABLETIME) { r.result.result.nanoSecPerRun = fastest; @@ -420,7 +428,7 @@ static BMK_return_t BMK_benchMemAdvancedNoAlloc( const void ** const srcPtrs, size_t* const srcSizes, void** const cPtrs, size_t* const cSizes, void** const resPtrs, size_t* const resSizes, - void* resultBuffer, void* compressedBuffer, + void** resultBufferPtr, void* compressedBuffer, const size_t maxCompressedSize, BMK_timedFnState_t* timeStateCompress, BMK_timedFnState_t* timeStateDecompress, @@ -432,7 +440,7 @@ static BMK_return_t BMK_benchMemAdvancedNoAlloc( int displayLevel, const char* displayName, const BMK_advancedParams_t* adv) { size_t const blockSize = ((adv->blockSize>=32 && (adv->mode != BMK_decodeOnly)) ? adv->blockSize : srcSize) + (!srcSize); /* avoid div by 0 */ - BMK_return_t results; + BMK_return_t results = { { 0, 0., 0., 0 }, 0 } ; size_t const loadedCompressedSize = srcSize; size_t cSize = 0; double ratio = 0.; @@ -454,13 +462,14 @@ static BMK_return_t BMK_benchMemAdvancedNoAlloc( srcPtr += fileSizes[fileNb]; } { size_t const decodedSize = (size_t)totalDSize64; - free(resultBuffer); - resultBuffer = malloc(decodedSize); - if (!resultBuffer) { + free(*resultBufferPtr); + //TODO: decodedSize corrupted? + *resultBufferPtr = malloc(decodedSize); + if (!(*resultBufferPtr)) { EXM_THROW(33, BMK_return_t, "not enough memory"); } if (totalDSize64 > decodedSize) { - free(resultBuffer); + free(*resultBufferPtr); EXM_THROW(32, BMK_return_t, "original size is too large"); /* size_t overflow */ } cSize = srcSize; @@ -472,7 +481,7 @@ static BMK_return_t BMK_benchMemAdvancedNoAlloc( /* Init data blocks */ { const char* srcPtr = (const char*)srcBuffer; char* cPtr = (char*)compressedBuffer; - char* resPtr = (char*)resultBuffer; + char* resPtr = (char*)(*resultBufferPtr); U32 fileNb; for (nbBlocks=0, fileNb=0; fileNbmode != BMK_decodeOnly) { + BMK_customReturn_t compressionResults = BMK_benchFunction(&local_defaultCompress, (void*)ctx, &local_initCCtx, (void*)&cctxprep, - nbBlocks, srcPtrs, srcSizes, cPtrs, cSizes, adv->nbSeconds); + nbBlocks, srcPtrs, srcSizes, cPtrs, cSizes, adv->nbSeconds); if(compressionResults.error) { results.error = compressionResults.error; return results; @@ -642,7 +652,8 @@ static BMK_return_t BMK_benchMemAdvancedNoAlloc( } /* CRC Checking */ - { U64 const crcCheck = XXH64(resultBuffer, srcSize, 0); + { void* resultBuffer = *resultBufferPtr; + U64 const crcCheck = XXH64(resultBuffer, srcSize, 0); /* adv->mode == 0 -> compress + decompress */ if ((adv->mode == BMK_both) && (crcOrig!=crcCheck)) { size_t u; @@ -692,11 +703,13 @@ static BMK_return_t BMK_benchMemAdvancedNoAlloc( } DISPLAYLEVEL(2, "%2i#\n", cLevel); } /* Bench */ - results.result.cMem = ZSTD_sizeof_CCtx(ctx); + results.result.cMem = (1 << (comprParams->windowLog)) + ZSTD_sizeof_CCtx(ctx); + results.error = 0; return results; } BMK_return_t BMK_benchMemAdvanced(const void* srcBuffer, size_t srcSize, + void* dstBuffer, size_t dstCapacity, const size_t* fileSizes, unsigned nbFiles, const int cLevel, const ZSTD_compressionParameters* comprParams, const void* dictBuffer, size_t dictBufferSize, @@ -717,26 +730,41 @@ BMK_return_t BMK_benchMemAdvanced(const void* srcBuffer, size_t srcSize, void ** const resPtrs = (void** const)malloc(maxNbBlocks * sizeof(void*)); size_t* const resSizes = (size_t* const)malloc(maxNbBlocks * sizeof(size_t)); - const size_t maxCompressedSize = ZSTD_compressBound(srcSize) + (maxNbBlocks * 1024); /* add some room for safety */ - void* compressedBuffer = malloc(maxCompressedSize); - void* resultBuffer = malloc(srcSize); BMK_timedFnState_t* timeStateCompress = BMK_createTimeState(adv->nbSeconds); BMK_timedFnState_t* timeStateDecompress = BMK_createTimeState(adv->nbSeconds); + void* compressedBuffer; + const size_t maxCompressedSize = dstCapacity ? dstCapacity : ZSTD_compressBound(srcSize) + (maxNbBlocks * 1024); + void* resultBuffer = malloc(srcSize); + + BMK_return_t results; - int allocationincomplete = !compressedBuffer || !resultBuffer || + int allocationincomplete; + + if(!dstCapacity) { + compressedBuffer = malloc(maxCompressedSize); + } else { + compressedBuffer = dstBuffer; + } + + allocationincomplete = !compressedBuffer || !resultBuffer || !srcPtrs || !srcSizes || !cPtrs || !cSizes || !resPtrs || !resSizes; + if (!allocationincomplete) { results = BMK_benchMemAdvancedNoAlloc(srcPtrs, srcSizes, cPtrs, cSizes, - resPtrs, resSizes, resultBuffer, compressedBuffer, maxCompressedSize, timeStateCompress, timeStateDecompress, + resPtrs, resSizes, &resultBuffer, compressedBuffer, maxCompressedSize, timeStateCompress, timeStateDecompress, srcBuffer, srcSize, fileSizes, nbFiles, cLevel, comprParams, dictBuffer, dictBufferSize, ctx, dctx, displayLevel, displayName, adv); } + + /* clean up */ BMK_freeTimeState(timeStateCompress); BMK_freeTimeState(timeStateDecompress); - free(compressedBuffer); + if(!dstCapacity) { /* only free if not given */ + free(compressedBuffer); + } free(resultBuffer); free((void*)srcPtrs); @@ -749,7 +777,6 @@ BMK_return_t BMK_benchMemAdvanced(const void* srcBuffer, size_t srcSize, if(allocationincomplete) { EXM_THROW(31, BMK_return_t, "allocation error : not enough memory"); } - results.error = 0; return results; } @@ -762,6 +789,7 @@ BMK_return_t BMK_benchMem(const void* srcBuffer, size_t srcSize, const BMK_advancedParams_t adv = BMK_initAdvancedParams(); return BMK_benchMemAdvanced(srcBuffer, srcSize, + NULL, 0, fileSizes, nbFiles, cLevel, comprParams, dictBuffer, dictBufferSize, @@ -783,6 +811,7 @@ static BMK_return_t BMK_benchMemCtxless(const void* srcBuffer, size_t srcSize, EXM_THROW(12, BMK_return_t, "not enough memory for contexts"); } res = BMK_benchMemAdvanced(srcBuffer, srcSize, + NULL, 0, fileSizes, nbFiles, cLevel, comprParams, dictBuffer, dictBufferSize, diff --git a/programs/bench.h b/programs/bench.h index f1ac255f8..625b65757 100644 --- a/programs/bench.h +++ b/programs/bench.h @@ -122,6 +122,8 @@ BMK_return_t BMK_syntheticTest(int cLevel, double compressibility, * (cLevel, comprParams + adv in advanced Mode) */ /* srcBuffer - data source, expected to be valid compressed data if in Decode Only Mode * srcSize - size of data in srcBuffer + * dstBuffer - destination buffer to write compressed output in, optional (NULL) + * dstCapacity - capacity of destination buffer, give 0 if dstBuffer = NULL * cLevel - compression level * comprParams - basic compression parameters * dictBuffer - a dictionary if used, null otherwise @@ -144,6 +146,7 @@ BMK_return_t BMK_benchMem(const void* srcBuffer, size_t srcSize, /* See benchMem for normal parameter uses and return, see advancedParams_t for adv */ BMK_return_t BMK_benchMemAdvanced(const void* srcBuffer, size_t srcSize, + void* dstBuffer, size_t dstCapacity, const size_t* fileSizes, unsigned nbFiles, const int cLevel, const ZSTD_compressionParameters* comprParams, const void* dictBuffer, size_t dictBufferSize, diff --git a/tests/Makefile b/tests/Makefile index 81e685780..ecf7fe2a7 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -200,7 +200,7 @@ zstreamtest-dll : $(ZSTDDIR)/common/xxhash.c # xxh symbols not exposed from dll zstreamtest-dll : $(ZSTREAM_LOCAL_FILES) $(CC) $(CPPFLAGS) $(CFLAGS) $(filter %.c,$^) $(LDFLAGS) -o $@$(EXT) -paramgrill : DEBUGFLAGS = # turn off assert() for speed measurements +#paramgrill : DEBUGFLAGS = # turn off assert() for speed measurements paramgrill : $(ZSTD_FILES) $(PRGDIR)/bench.c $(PRGDIR)/datagen.c paramgrill.c $(CC) $(FLAGS) $^ -lm -o $@$(EXT) diff --git a/tests/paramgrill.c b/tests/paramgrill.c index 81c6dffc2..44bcedef8 100644 --- a/tests/paramgrill.c +++ b/tests/paramgrill.c @@ -41,6 +41,8 @@ #define MB *(1<<20) #define GB *(1ULL<<30) +#define TIMELOOP_NANOSEC (1*1000000000ULL) /* 1 second */ + #define NBLOOPS 2 #define TIMELOOP (2 * SEC_TO_MICRO) #define NB_LEVELS_TRACKED 22 /* ensured being >= ZSTD_maxCLevel() in BMK_init_level_constraints() */ @@ -70,13 +72,39 @@ static const int g_maxNbVariations = 64; #define SLOG_IND 3 #define SLEN_IND 4 #define TLEN_IND 5 -#define STRT_IND 6 -#define NUM_PARAMS 7 +//#define STRT_IND 6 +//#define NUM_PARAMS 7 +#define NUM_PARAMS 6 +//just don't use strategy as a param. +#undef ZSTD_WINDOWLOG_MAX +#define ZSTD_WINDOWLOG_MAX 27 //no long range stuff for now. + +//make 2^[0,10] w/ 999 +#define ZSTD_TARGETLENGTH_MIN 0 //actually targeLengthlog min +#define ZSTD_TARGETLENGTH_MAX 10 + +//#define ZSTD_TARGETLENGTH_MAX 1024 +#define WLOG_RANGE (ZSTD_WINDOWLOG_MAX - ZSTD_WINDOWLOG_MIN + 1) +#define CLOG_RANGE (ZSTD_CHAINLOG_MAX - ZSTD_CHAINLOG_MIN + 1) +#define HLOG_RANGE (ZSTD_HASHLOG_MAX - ZSTD_HASHLOG_MIN + 1) +#define SLOG_RANGE (ZSTD_SEARCHLOG_MAX - ZSTD_SEARCHLOG_MIN + 1) +#define SLEN_RANGE (ZSTD_SEARCHLENGTH_MAX - ZSTD_SEARCHLENGTH_MIN + 1) +#define TLEN_RANGE 11 +//hard coded since we only use powers of 2 (and 999 ~ 1024) + +static const int mintable[NUM_PARAMS] = { ZSTD_WINDOWLOG_MIN, ZSTD_CHAINLOG_MIN, ZSTD_HASHLOG_MIN, ZSTD_SEARCHLOG_MIN, ZSTD_SEARCHLENGTH_MIN, ZSTD_TARGETLENGTH_MIN }; +static const int maxtable[NUM_PARAMS] = { ZSTD_WINDOWLOG_MAX, ZSTD_CHAINLOG_MAX, ZSTD_HASHLOG_MAX, ZSTD_SEARCHLOG_MAX, ZSTD_SEARCHLENGTH_MAX, ZSTD_TARGETLENGTH_MAX }; +static const int rangetable[NUM_PARAMS] = { WLOG_RANGE, CLOG_RANGE, HLOG_RANGE, SLOG_RANGE, SLEN_RANGE, TLEN_RANGE }; + +//use grid-search or something when space is small enough? +#define SMALL_SEARCH_SPACE 1000 /*-************************************ * Benchmark Parameters **************************************/ +typedef BYTE U8; + static double g_grillDuration_s = 99999; /* about 27 hours */ static U32 g_nbIterations = NBLOOPS; static double g_compressibility = COMPRESSIBILITY_DEFAULT; @@ -150,13 +178,74 @@ static unsigned longCommandWArg(const char** stringPtr, const char* longCommand) return result; } +//assume that clock can at least measure .01 second intervals? +//make this a settable global initialized with fn? +//#define CLOCK_GRANULARITY 100000000ULL +static U64 g_clockGranularity = 100000000ULL; + +static void findClockGranularity(void) { + UTIL_time_t clockStart = UTIL_getTime(); + U64 el1 = 0, el2 = 0; + int i = 0; + do { + el1 = el2; + el2 = UTIL_clockSpanNano(clockStart); + if(el1 < el2) { + U64 iv = el2 - el1; + if(g_clockGranularity > iv) { + g_clockGranularity = iv; + i = 0; + } else { + i++; + } + } + } while(i < 10); + DISPLAY("Granularity: %llu\n", (unsigned long long)g_clockGranularity); +} typedef struct { U32 cSpeed; /* bytes / sec */ U32 dSpeed; - U32 Mem; /* bytes */ + U32 cMem; /* bytes */ } constraint_t; +#define CLAMPCHECK(val,min,max) { \ + if (val && (((val)<(min)) | ((val)>(max)))) { \ + DISPLAY("INVALID PARAMETER CONSTRAINTS\n"); \ + return 0; \ +} } + + +/* Like ZSTD_checkCParams() but allows 0's */ +/* no check on targetLen? */ +static int cParamValid(ZSTD_compressionParameters paramTarget) { + CLAMPCHECK(paramTarget.hashLog, ZSTD_HASHLOG_MIN, ZSTD_HASHLOG_MAX); + CLAMPCHECK(paramTarget.searchLog, ZSTD_SEARCHLOG_MIN, ZSTD_SEARCHLOG_MAX); + CLAMPCHECK(paramTarget.searchLength, ZSTD_SEARCHLENGTH_MIN, ZSTD_SEARCHLENGTH_MAX); + CLAMPCHECK(paramTarget.windowLog, ZSTD_WINDOWLOG_MIN, ZSTD_WINDOWLOG_MAX); + CLAMPCHECK(paramTarget.chainLog, ZSTD_CHAINLOG_MIN, ZSTD_CHAINLOG_MAX); + if(paramTarget.strategy > ZSTD_btultra) { + DISPLAY("INVALID PARAMETER CONSTRAINTS\n"); + return 0; + } + return 1; +} + +static void cParamZeroMin(ZSTD_compressionParameters* paramTarget) { + paramTarget->windowLog = paramTarget->windowLog ? paramTarget->windowLog : ZSTD_WINDOWLOG_MIN; + paramTarget->searchLog = paramTarget->searchLog ? paramTarget->searchLog : ZSTD_SEARCHLOG_MIN; + paramTarget->chainLog = paramTarget->chainLog ? paramTarget->chainLog : ZSTD_CHAINLOG_MIN; + paramTarget->hashLog = paramTarget->hashLog ? paramTarget->hashLog : ZSTD_HASHLOG_MIN; + paramTarget->searchLength = paramTarget->searchLength ? paramTarget->searchLength : ZSTD_SEARCHLENGTH_MIN; + paramTarget->targetLength = paramTarget->targetLength ? paramTarget->targetLength : 1; +} + +static void BMK_translateAdvancedParams(ZSTD_compressionParameters params) +{ + DISPLAY("--zstd=windowLog=%u,chainLog=%u,hashLog=%u,searchLog=%u,searchLength=%u,targetLength=%u,strategy=%u \n", + params.windowLog, params.chainLog, params.hashLog, params.searchLog, params.searchLength, params.targetLength, (U32)(params.strategy)); +} + /*-******************************************************* * Bench functions *********************************************************/ @@ -357,20 +446,45 @@ static int BMK_seed(winnerInfo_t* winners, const ZSTD_compressionParameters para return better; } +/* bounds check in sanitize too? */ +#define CLAMP(var, lo, hi) { \ + var = MAX(MIN(var, hi), lo); \ +} + /* nullified useless params, to ensure count stats */ -static ZSTD_compressionParameters* sanitizeParams(ZSTD_compressionParameters params) +/* no point in windowLog < chainLog (no point 2x chainLog for bt) */ +/* now with built in bounds-checking */ +/* no longer does anything with sanitizeVarArray + clampcheck */ +static ZSTD_compressionParameters sanitizeParams(ZSTD_compressionParameters params) { - g_params = params; if (params.strategy == ZSTD_fast) g_params.chainLog = 0, g_params.searchLog = 0; if (params.strategy == ZSTD_dfast) g_params.searchLog = 0; - if (params.strategy != ZSTD_btopt && params.strategy != ZSTD_btultra) + if (params.strategy != ZSTD_btopt && params.strategy != ZSTD_btultra && params.strategy != ZSTD_fast) g_params.targetLength = 0; - return &g_params; + + return params; +} + +/* new length */ +/* keep old array, will need if iter over strategy. */ +static int sanitizeVarArray(int varLength, U32* varArray, U32* varNew, ZSTD_strategy strat) { + int i, j = 0; + for(i = 0; i < varLength; i++) { + if( !((varArray[i] == CLOG_IND && strat == ZSTD_fast) + || (varArray[i] == SLOG_IND && strat == ZSTD_dfast) + || (varArray[i] == TLEN_IND && strat != ZSTD_btopt && strat != ZSTD_btultra && strat != ZSTD_fast))) { + varNew[j] = varArray[i]; + j++; + } + } + return j; + } /* res should be NUM_PARAMS size */ +/* constructs varArray from ZSTD_compressionParameters style parameter */ static int variableParams(const ZSTD_compressionParameters paramConstraints, U32* res) { int j = 0; if(!paramConstraints.windowLog) { @@ -397,10 +511,6 @@ static int variableParams(const ZSTD_compressionParameters paramConstraints, U32 res[j] = TLEN_IND; j++; } - if(!(U32)paramConstraints.strategy) { - res[j] = STRT_IND; - j++; - } return j; } @@ -417,43 +527,39 @@ static int inverseVariableParams(const ZSTD_compressionParameters paramConstrain res[j] = CLOG_IND; j++; } else { - res[WLOG_IND] = -1; + res[CLOG_IND] = -1; } if(!paramConstraints.hashLog) { res[j] = HLOG_IND; j++; } else { - res[WLOG_IND] = -1; + res[HLOG_IND] = -1; } if(!paramConstraints.searchLog) { res[j] = SLOG_IND; j++; } else { - res[WLOG_IND] = -1; + res[SLOG_IND] = -1; } if(!paramConstraints.searchLength) { res[j] = SLEN_IND; j++; } else { - res[WLOG_IND] = -1; + res[SLEN_IND] = -1; } if(!paramConstraints.targetLength) { res[j] = TLEN_IND; j++; } else { - res[WLOG_IND] = -1; - } - if(!(U32)paramConstraints.strategy) { - res[j] = STRT_IND; - j++; - } else { - res[WLOG_IND] = -1; + res[TLEN_IND] = -1; } + return j; } /* amt will probably always be \pm 1? */ /* slight change from old paramVariation, targetLength can only take on powers of 2 now (999 ~= 1024?) */ +/* take max/min bounds into account as well? */ static void paramVaryOnce(U32 paramIndex, int amt, ZSTD_compressionParameters* ptr) { switch(paramIndex) { @@ -463,13 +569,16 @@ static void paramVaryOnce(U32 paramIndex, int amt, ZSTD_compressionParameters* p case SLOG_IND: ptr->searchLog += amt; break; case SLEN_IND: ptr->searchLength += amt; break; case TLEN_IND: - if(amt > 0) { + if(amt >= 0) { ptr->targetLength <<= amt; + ptr->targetLength = MIN(ptr->targetLength, 999); } else { + if(ptr->targetLength == 999) { + ptr->targetLength = 1024; + } ptr->targetLength >>= -amt; } break; - case STRT_IND: ptr->strategy += amt; break; default: break; } } @@ -477,34 +586,143 @@ static void paramVaryOnce(U32 paramIndex, int amt, ZSTD_compressionParameters* p //Don't fuzz fixed variables. //turn pcs to pcs array with macro for params. //pass in variation array from variableParams -static void paramVariation(ZSTD_compressionParameters* ptr, const U32* varyParams, const int varyLen) +//take nbChanges as argument? +static void paramVariation(ZSTD_compressionParameters* ptr, const U32* varyParams, const int varyLen, U32 nbChanges) { ZSTD_compressionParameters p; U32 validated = 0; while (!validated) { - U32 nbChanges = (FUZ_rand(&g_rand) & 3) + 1; + U32 i; p = *ptr; - for ( ; nbChanges ; nbChanges--) { - const U32 changeID = FUZ_rand(&g_rand) % (2 * varyLen); + for (i = 0 ; i < nbChanges ; i++) { + const U32 changeID = FUZ_rand(&g_rand) % (varyLen << 1); paramVaryOnce(varyParams[changeID >> 1], ((changeID & 1) << 1) - 1, &p); } - validated = !ZSTD_isError(ZSTD_checkCParams(p)); - + //validated = !ZSTD_isError(ZSTD_checkCParams(p)); + validated = cParamValid(p); + //Make sure memory is at least close to feasible? //ZSTD_estimateCCtxSize thing. } - *ptr = p; + *ptr = sanitizeParams(p); } +//varyParams gives us table size? +//1 per strategy +//varyParams should always be sorted smallest to largest +//take arrayLen to allocate memotable +//should be ~10^7 unconstrained. +static size_t memoTableLen(const U32* varyParams, const int varyLen) { + size_t arrayLen = 1; + int i; + for(i = 0; i < varyLen; i++) { + arrayLen *= rangetable[varyParams[i]]; + } + return arrayLen; +} + +//sort of ~lg2 (replace 1024 w/ 999) for memoTableInd Tlen +static unsigned lg2(unsigned x) { + unsigned j = 0; + if(x == 999) { + return 10; + } + while(x >>= 1) { + j++; + } + return j; +} + +//indexes compressionParameters into memotable +//of form +static unsigned memoTableInd(const ZSTD_compressionParameters* ptr, const U32* varyParams, const int varyLen) { + int i; + unsigned ind = 0; + for(i = 0; i < varyLen; i++) { + switch(varyParams[i]) { + case WLOG_IND: ind *= WLOG_RANGE; ind += ptr->windowLog - ZSTD_WINDOWLOG_MIN ; break; + case CLOG_IND: ind *= CLOG_RANGE; ind += ptr->chainLog - ZSTD_CHAINLOG_MIN ; break; + case HLOG_IND: ind *= HLOG_RANGE; ind += ptr->hashLog - ZSTD_HASHLOG_MIN ; break; + case SLOG_IND: ind *= SLOG_RANGE; ind += ptr->searchLog - ZSTD_SEARCHLOG_MIN ; break; + case SLEN_IND: ind *= SLEN_RANGE; ind += ptr->searchLength - ZSTD_SEARCHLENGTH_MIN; break; + case TLEN_IND: ind *= TLEN_RANGE; ind += lg2(ptr->targetLength) - ZSTD_TARGETLENGTH_MIN; break; + } + } + return ind; +} + +/* presumably, the unfilled parameters are already at their correct value */ +/* inverse above function for varyParams */ +static void memoTableIndInv(ZSTD_compressionParameters* ptr, const U32* varyParams, const int varyLen, size_t ind) { + int i; + for(i = varyLen - 1; i >= 0; i--) { + switch(varyParams[i]) { + case WLOG_IND: ptr->windowLog = ind % WLOG_RANGE + ZSTD_WINDOWLOG_MIN; ind /= WLOG_RANGE; break; + case CLOG_IND: ptr->chainLog = ind % CLOG_RANGE + ZSTD_CHAINLOG_MIN; ind /= CLOG_RANGE; break; + case HLOG_IND: ptr->hashLog = ind % HLOG_RANGE + ZSTD_HASHLOG_MIN; ind /= HLOG_RANGE; break; + case SLOG_IND: ptr->searchLog = ind % SLOG_RANGE + ZSTD_SEARCHLOG_MIN; ind /= SLOG_RANGE; break; + case SLEN_IND: ptr->searchLength = ind % SLEN_RANGE + ZSTD_SEARCHLENGTH_MIN; ind /= SLEN_RANGE; break; + case TLEN_IND: ptr->targetLength = MIN(1 << (ind % TLEN_RANGE), 999); ind /= TLEN_RANGE; break; + } + } +} + +//initializing memoTable +/* */ +static void memoTableInit(U8* memoTable, ZSTD_compressionParameters paramConstraints, constraint_t target, const U32* varyParams, const int varyLen) { + size_t i; + size_t arrayLen = memoTableLen(varyParams, varyLen); + int cwFixed = !paramConstraints.chainLog || !paramConstraints.windowLog; + int scFixed = !paramConstraints.searchLog || !paramConstraints.chainLog; + int j = 0; + memset(memoTable, 0, arrayLen); + + + for(i = 0; i < arrayLen; i++) { + memoTableIndInv(¶mConstraints, varyParams, varyLen, i); + BMK_translateAdvancedParams(paramConstraints); + if(ZSTD_estimateCCtxSize_usingCParams(paramConstraints) + (1 << paramConstraints.windowLog) > target.cMem) { + //infeasible; + memoTable[i] = 255; + j++; + } + /* nil out parameter sets equivalent to others. */ + if(cwFixed/* at most least 1 param fixed. */) { + if(paramConstraints.strategy == ZSTD_btlazy2 || paramConstraints.strategy == ZSTD_btopt || paramConstraints.strategy == ZSTD_btultra) { + if(paramConstraints.chainLog > paramConstraints.windowLog + 1) { + if(memoTable[i] != 255) { j++; } + memoTable[i] = 255; + } + } else { + if(paramConstraints.chainLog > paramConstraints.windowLog) { + if(memoTable[i] != 255) { j++; } + memoTable[i] = 255; + } + } + } + if(scFixed) { + if(paramConstraints.searchLog > paramConstraints.chainLog) { + if(memoTable[i] != 255) { j++; } + memoTable[i] = 255; + } + } + } + DISPLAY("%d / %d Invalid\n", j, (int)i); +} #define PARAMTABLELOG 25 #define PARAMTABLESIZE (1<> 3) & PARAMTABLEMASK] + g_alreadyTested[(XXH64(((void*)&sanitizeParams(p), sizeof(p), 0) >> 3) & PARAMTABLEMASK] */ +static BYTE* NB_TESTS_PLAYED(ZSTD_compressionParameters p) { + ZSTD_compressionParameters p2 = sanitizeParams(p); + return &g_alreadyTested[(XXH64((void*)&p2, sizeof(p2), 0) >> 3) & PARAMTABLEMASK]; +} static void playAround(FILE* f, winnerInfo_t* winners, ZSTD_compressionParameters params, @@ -513,21 +731,23 @@ static void playAround(FILE* f, winnerInfo_t* winners, { int nbVariations = 0; UTIL_time_t const clockStart = UTIL_getTime(); - const U32 unconstrained[NUM_PARAMS] = { 0, 1, 2, 3, 4, 5, 6 }; + const U32 unconstrained[NUM_PARAMS] = { 0, 1, 2, 3, 4, 5 }; while (UTIL_clockSpanMicro(clockStart) < g_maxVariationTime) { ZSTD_compressionParameters p = params; + BYTE* b; if (nbVariations++ > g_maxNbVariations) break; - paramVariation(&p, unconstrained, 7); + paramVariation(&p, unconstrained, 7, 4); /* exclude faster if already played params */ - if (FUZ_rand(&g_rand) & ((1 << NB_TESTS_PLAYED(p))-1)) + if (FUZ_rand(&g_rand) & ((1 << *NB_TESTS_PLAYED(p))-1)) continue; /* test */ - NB_TESTS_PLAYED(p)++; + b = NB_TESTS_PLAYED(p); + (*b)++; if (!BMK_seed(winners, p, srcBuffer, srcSize, ctx, dctx)) continue; /* improvement found => search more */ @@ -544,34 +764,37 @@ static ZSTD_compressionParameters randomParams(void) U32 validated = 0; while (!validated) { /* totally random entry */ - p.chainLog = (FUZ_rand(&g_rand) % (ZSTD_CHAINLOG_MAX+1 - ZSTD_CHAINLOG_MIN)) + ZSTD_CHAINLOG_MIN; - p.hashLog = (FUZ_rand(&g_rand) % (ZSTD_HASHLOG_MAX+1 - ZSTD_HASHLOG_MIN)) + ZSTD_HASHLOG_MIN; - p.searchLog = (FUZ_rand(&g_rand) % (ZSTD_SEARCHLOG_MAX+1 - ZSTD_SEARCHLOG_MIN)) + ZSTD_SEARCHLOG_MIN; - p.windowLog = (FUZ_rand(&g_rand) % (ZSTD_WINDOWLOG_MAX+1 - ZSTD_WINDOWLOG_MIN)) + ZSTD_WINDOWLOG_MIN; + p.chainLog = (FUZ_rand(&g_rand) % (ZSTD_CHAINLOG_MAX+1 - ZSTD_CHAINLOG_MIN)) + ZSTD_CHAINLOG_MIN; + p.hashLog = (FUZ_rand(&g_rand) % (ZSTD_HASHLOG_MAX+1 - ZSTD_HASHLOG_MIN)) + ZSTD_HASHLOG_MIN; + p.searchLog = (FUZ_rand(&g_rand) % (ZSTD_SEARCHLOG_MAX+1 - ZSTD_SEARCHLOG_MIN)) + ZSTD_SEARCHLOG_MIN; + p.windowLog = (FUZ_rand(&g_rand) % (ZSTD_WINDOWLOG_MAX+1 - ZSTD_WINDOWLOG_MIN)) + ZSTD_WINDOWLOG_MIN; p.searchLength=(FUZ_rand(&g_rand) % (ZSTD_SEARCHLENGTH_MAX+1 - ZSTD_SEARCHLENGTH_MIN)) + ZSTD_SEARCHLENGTH_MIN; p.targetLength=(FUZ_rand(&g_rand) % (512)); p.strategy = (ZSTD_strategy) (FUZ_rand(&g_rand) % (ZSTD_btultra +1)); - validated = !ZSTD_isError(ZSTD_checkCParams(p)); + //validated = !ZSTD_isError(ZSTD_checkCParams(p)); + validated = cParamValid(p); } return p; } -static ZSTD_compressionParameters randomConstrainedParams(ZSTD_compressionParameters pc) +//destructively modifies pc. +//Maybe if memoTable[ind] > 0 too often, count zeroes and explicitly choose from free stuff? +//^ maybe this doesn't matter, with |mt| size it has \approx 1-(1/e) of finding even single free spot in |mt| tries, not too bad. +//TODO: maybe memoTable pc before sanitization too so no repeats? +static void randomConstrainedParams(ZSTD_compressionParameters* pc, U32* varArray, int varLen, U8* memoTable) { - ZSTD_compressionParameters p; - U32 validated = 0; - while (!validated) { - /* totally random entry */ - if(!pc.chainLog) p.chainLog = (FUZ_rand(&g_rand) % (ZSTD_CHAINLOG_MAX+1 - ZSTD_CHAINLOG_MIN)) + ZSTD_CHAINLOG_MIN; - if(!pc.chainLog) p.hashLog = (FUZ_rand(&g_rand) % (ZSTD_HASHLOG_MAX+1 - ZSTD_HASHLOG_MIN)) + ZSTD_HASHLOG_MIN; - if(!pc.chainLog) p.searchLog = (FUZ_rand(&g_rand) % (ZSTD_SEARCHLOG_MAX+1 - ZSTD_SEARCHLOG_MIN)) + ZSTD_SEARCHLOG_MIN; - if(!pc.chainLog) p.windowLog = (FUZ_rand(&g_rand) % (ZSTD_WINDOWLOG_MAX+1 - ZSTD_WINDOWLOG_MIN)) + ZSTD_WINDOWLOG_MIN; - if(!pc.chainLog) p.searchLength=(FUZ_rand(&g_rand) % (ZSTD_SEARCHLENGTH_MAX+1 - ZSTD_SEARCHLENGTH_MIN)) + ZSTD_SEARCHLENGTH_MIN; - if(!pc.chainLog) p.targetLength=(FUZ_rand(&g_rand) % (512)) + 1; //ZSTD_TARGETLENGTH_MIN; //change to 2^[0,10?] - if(!pc.chainLog) p.strategy = (ZSTD_strategy) (FUZ_rand(&g_rand) % (ZSTD_btultra +1)); - validated = !ZSTD_isError(ZSTD_checkCParams(p)); - } - return p; + int tries = memoTableLen(varArray, varLen); //configurable, + const size_t maxSize = memoTableLen(varArray, varLen); + size_t ind; + do { + ind = (FUZ_rand(&g_rand)) % maxSize; + tries--; + } while(memoTable[ind] > 0 && tries > 0); + //&& FUZ_rand(&g_rand) % 256 > memoTable[ind]); get nd choosing? (helpful w/ distance) /* maybe > infeasible bound? */ + + /* memoTable[ind] == 0 -> unexplored */ + memoTableIndInv(pc, varArray, varLen, (unsigned)ind); + *pc = sanitizeParams(*pc); } static void BMK_selectRandomStart( @@ -746,22 +969,80 @@ int benchFiles(const char** fileNamesTable, int nbFiles) return 0; } - -static void BMK_translateAdvancedParams(ZSTD_compressionParameters params) -{ - DISPLAY("--zstd=windowLog=%u,chainLog=%u,hashLog=%u,searchLog=%u,searchLength=%u,targetLength=%u,strategy=%u \n", - params.windowLog, params.chainLog, params.hashLog, params.searchLog, params.searchLength, params.targetLength, (U32)(params.strategy)); -} - -//Results currently don't capture memory usage or anything. //parameter feasibility is not checked, should just be restricted from use. static int feasible(BMK_result_t results, constraint_t target) { - return (results.cSpeed >= target.cSpeed) && (results.dSpeed >= target.dSpeed) && (results.cMem <= target.Mem || !target.Mem); + return (results.cSpeed >= target.cSpeed) && (results.dSpeed >= target.dSpeed) && (results.cMem <= target.cMem || !target.cMem); +} + +#define EPSILON 0.01 +static int epsilonEqual(double c1, double c2) { + return MAX(c1/c2,c2/c1) < 1 + EPSILON; +} + +//so the compiler stops warning +static int eqZero(double c1) { + return (U64)c1 == (U64)0.0 || (U64)c1 == (U64)-0.0; } /* returns 1 if result2 is strictly 'better' than result1 */ +/* strict comparison / cutoff based */ static int objective_lt(BMK_result_t result1, BMK_result_t result2) { - return (result1.cSize > result2.cSize) || (result1.cSize == result2.cSize && result2.cSpeed > result1.cSpeed); + return (result1.cSize > result2.cSize) || (epsilonEqual(result1.cSize, result2.cSize) && result2.cSpeed > result1.cSpeed) + || (epsilonEqual(result1.cSize,result2.cSize) && epsilonEqual(result2.cSpeed, result1.cSpeed) && result2.dSpeed > result1.dSpeed); +} + +//will probably be some linear combinartion of comp speed, decompSpeed, & ratio (maybe size), and memory? +//pretty arbitrary right now +//maybe better - higher coefficient when below threshold, lower when above +//need to normalize speed? or just use ratio speed / target? +//Maybe don't use ratio at all when looking for feasibility? + +/* maybe dynamically vary the coefficients for this around based on what's already been discovered. (maybe make a reversed ratio cutoff?) concave to pheaily penalize below ratio? */ +static double resultScore(BMK_result_t res, size_t srcSize, constraint_t target) { + double cs = 0., ds = 0., rt, cm = 0.; + const double r1 = 1, r2 = 0.1, rtr = 0.5; + double ret; + if(target.cSpeed) { cs = res.cSpeed / (double)target.cSpeed; } + if(target.dSpeed) { ds = res.dSpeed / (double)target.dSpeed; } + if(target.cMem != (U32)-1) { cm = (double)target.cMem / res.cMem; } + rt = ((double)srcSize / res.cSize); + + //(void)rt; + //(void)rtr; + + ret = (MIN(1, cs) + MIN(1, ds) + MIN(1, cm))*r1 + rt * rtr + + (MAX(0, log(cs))+ MAX(0, log(ds))+ MAX(0, log(cm))) * r2; + //DISPLAY("resultScore: %f\n", ret); + return ret; +} + +/* + double W_ratio = (double)srcSize / testResult.cSize; + double O_ratio = (double)srcSize / winners[cLevel].result.cSize; + double W_ratioNote = log (W_ratio); + double O_ratioNote = log (O_ratio); + size_t W_DMemUsed = (1 << params.windowLog) + (16 KB); + size_t O_DMemUsed = (1 << winners[cLevel].params.windowLog) + (16 KB); + double W_DMemUsed_note = W_ratioNote * ( 40 + 9*cLevel) - log((double)W_DMemUsed); + double O_DMemUsed_note = O_ratioNote * ( 40 + 9*cLevel) - log((double)O_DMemUsed); + + size_t W_CMemUsed = (1 << params.windowLog) + ZSTD_estimateCCtxSize_usingCParams(params); + size_t O_CMemUsed = (1 << winners[cLevel].params.windowLog) + ZSTD_estimateCCtxSize_usingCParams(winners[cLevel].params); + double W_CMemUsed_note = W_ratioNote * ( 50 + 13*cLevel) - log((double)W_CMemUsed); + double O_CMemUsed_note = O_ratioNote * ( 50 + 13*cLevel) - log((double)O_CMemUsed); + + double W_CSpeed_note = W_ratioNote * ( 30 + 10*cLevel) + log(testResult.cSpeed); + double O_CSpeed_note = O_ratioNote * ( 30 + 10*cLevel) + log(winners[cLevel].result.cSpeed); + + double W_DSpeed_note = W_ratioNote * ( 20 + 2*cLevel) + log(testResult.dSpeed); + double O_DSpeed_note = O_ratioNote * ( 20 + 2*cLevel) + log(winners[cLevel].result.dSpeed); + +*/ +//ratio tradeoffs, may be useful in guiding + +/* objective_lt, but based on scoring function */ +static int objective_lt2(BMK_result_t result1, BMK_result_t result2, size_t srcSize, constraint_t target) { + return resultScore(result1, srcSize, target) < resultScore(result2, srcSize, target); } /* res gives array dimensions, should be size NUM_PARAMS */ @@ -783,48 +1064,800 @@ static size_t computeStateSize(const ZSTD_compressionParameters paramConstraints static unsigned calcViolation(BMK_result_t results, constraint_t target) { int diffcSpeed = MAX(target.cSpeed - results.cSpeed, 0); int diffdSpeed = MAX(target.dSpeed - results.dSpeed, 0); - int diffcMem = MAX(results.cMem - target.Mem, 0); + int diffcMem = MAX(results.cMem - target.cMem, 0); return diffcSpeed + diffdSpeed + diffcMem; } -/* finds some set of parameters which fulfills req's - * Prioritize highest / try to locally minimize sum? - * Is it ever useful to go out of the param constraints? ? - * random / perturb when revisit? - * momentum? - */ -static ZSTD_compressionParameters findFeasible(constraint_t target, ZSTD_compressionParameters paramTarget) { - unsigned violation; - - ZSTD_compressionParameters winner = randomConstrainedParams(paramTarget); - //just use g_alreadyTested and xxhash? - BYTE* memotable; - do { - //prioritize memory - if(diffcMem >= diffcSpeed && diffcMem >= diffdSpeed) { - - //prioritize compression Speed - } else if (diffcSpeeed >= diffdSpeed && diffcSpeed >= diffcMem) { - - //prioritize decompressionSpeed - } else { - - } - violation = calcViolation(result, target); - } while(objective); - if(validate) { - DISPLAY("Feasible Point Found\n"); - return winner; +/* +uncertaintyConstant >= 1 +returns -1 = 'certainly' infeasible + 0 = unceratin + 1 = 'certainly' feasible +*/ +//paramTarget misnamed, should just be target +static int uncertainFeasibility(double const uncertaintyConstantC, double const uncertaintyConstantD, const constraint_t paramTarget, const BMK_result_t* const results) { + if((paramTarget.cSpeed != 0 && results->cSpeed * uncertaintyConstantC < paramTarget.cSpeed) || + (paramTarget.dSpeed != 0 && results->dSpeed * uncertaintyConstantD < paramTarget.dSpeed) || + (paramTarget.cMem != 0 && results->cMem > paramTarget.cMem)) { + return -1; + } else if((paramTarget.cSpeed == 0 || results->cSpeed / uncertaintyConstantC > paramTarget.cSpeed) && + (paramTarget.dSpeed == 0 || results->dSpeed / uncertaintyConstantD > paramTarget.dSpeed) && + (paramTarget.cMem == 0 || results->cMem <= paramTarget.cMem)) { + return 1; } else { - DISPLAY("No solution found\n"); - ZSTD_compressionParameters ret = { 0, 0, 0, 0, 0, 0, 0 }; - return ret; + return 0; } } +/* 1 - better than prev best + 0 - uncertain + -1 - worse + assume prev_best status is run fully? + but then we'd have to rerun any winners anyway */ +//presumably memory has already been compared, mostly worried about mem, cspeed, dspeed +//uncertainty only applies to speed. +//if using objective fn, this could be much easier since we could just scale that. +//difficult to make judgements about later parameters in prioritization type when there's +//uncertainty on the first. +static int uncertainComparison(double const uncertaintyConstantC, double const uncertaintyConstantD, BMK_result_t* candidate, BMK_result_t* prevBest) { + (void)uncertaintyConstantD; //unused for now + if(candidate->cSpeed > prevBest->cSpeed * uncertaintyConstantC) { + return 1; + } else if (candidate->cSpeed * uncertaintyConstantC < prevBest->cSpeed) { + return -1; + } else { + return 0; + } +} + +/* speed in b, srcSize in b/s loopDuration in ns */ +//TODO: simplify code in feasibleBench with this instead of writing it all out. +//only applicable for single loop +static double calcUncertainty(double speed, size_t srcSize) { + U64 loopDuration; + if(eqZero(speed)) { return 2; } + loopDuration = ((srcSize * TIMELOOP_NANOSEC) / speed); + return MIN((loopDuration + (double)2 * g_clockGranularity) / loopDuration, 2); +} + +//benchmarks and tests feasibility together +//1 = true = better +//0 = false = not better +//if true then resultPtr will give results. +//2+ on error? +//alt: error = 0 / infeasible as well; +//maybe use compress_only mode for ratio-finding benchmark? +//prioritize ratio > cSpeed > dSpeed > cMem +//Misnamed - should be worse, better, error +//alternative (to make work for feasible-pt searching as well) - only compare to winner, not to target +//but then we need to judge what better means in this context, which shouldn't be the same (strict ratio improvement) +#define INFEASIBLE_RESULT 0 +#define FEASIBLE_RESULT 1 +#define ERROR_RESULT 2 +static int feasibleBench(BMK_result_t* resultPtr, + const void* srcBuffer, size_t srcSize, + void* dstBuffer, size_t dstSize, + ZSTD_CCtx* ctx, ZSTD_DCtx* dctx, + const ZSTD_compressionParameters cParams, + const constraint_t target, + BMK_result_t* winnerResult) { + BMK_advancedParams_t adv = BMK_initAdvancedParams(); + BMK_return_t benchres; + U64 loopDurationC = 0, loopDurationD = 0; + double uncertaintyConstantC, uncertaintyConstantD; + adv.loopMode = BMK_iterMode; + adv.nbSeconds = 1; //get ratio and 2x approx speed? + + //alternative - test 1 iter for ratio, (possibility of error 3 which is fine), + //maybe iter this until 2x measurable for better guarantee? + DISPLAY("Feas:\n"); + benchres = BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, &srcSize, 1, 0, &cParams, NULL, 0, ctx, dctx, 0, "File", &adv); + if(benchres.error) { + DISPLAY("ERROR %d !!\n", benchres.error); + } + BMK_printWinner(stdout, CUSTOM_LEVEL, benchres.result, cParams, srcSize); + + if(!benchres.error) { + *resultPtr = benchres.result; + /* if speed is 0 (only happens when time = 0) */ + if(eqZero(benchres.result.cSpeed)) { + loopDurationC = 0; + uncertaintyConstantC = 2; + } else { + loopDurationC = ((srcSize * TIMELOOP_NANOSEC) / benchres.result.cSpeed); + //problem - tested in fullbench, saw speed vary 3x between iters, maybe raise uncertaintyConstraint up? + //possibly has to do with initCCtx? or system stuff? + //asymmetric +/- constant needed? + uncertaintyConstantC = MIN((loopDurationC + (double)(2 * g_clockGranularity)/loopDurationC), 2); //.02 seconds + } + if(eqZero(benchres.result.dSpeed)) { + loopDurationD = 0; + uncertaintyConstantD = 2; + } else { + loopDurationD = ((srcSize * TIMELOOP_NANOSEC) / benchres.result.dSpeed); + //problem - tested in fullbench, saw speed vary 3x between iters, maybe raise uncertaintyConstraint up? + //possibly has to do with initCCtx? or system stuff? + //asymmetric +/- constant needed? + uncertaintyConstantD = MIN((loopDurationD + (double)(2 * g_clockGranularity)/loopDurationD), 2); //.02 seconds + } + + + if(benchres.result.cSize < winnerResult->cSize) { //better compression ratio, just needs to be feasible + //optimistic assume speed + //incoporate some sort of tradeoff comparison with the winner's results? + int feas = uncertainFeasibility(uncertaintyConstantC, uncertaintyConstantD, target, &(benchres.result)); + if(feas == 0) { // uncertain feasibility + adv.loopMode = BMK_timeMode; + if(loopDurationC < TIMELOOP_NANOSEC) { + BMK_return_t benchres2; + adv.mode = BMK_compressOnly; + benchres2 = BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, &srcSize, 1, 0, &cParams, NULL, 0, ctx, dctx, 0, "File", &adv); + if(benchres2.error) { + return ERROR_RESULT; + } else { + benchres.result.cSpeed = benchres2.result.cSpeed; + } + } + if(loopDurationD < TIMELOOP_NANOSEC) { + BMK_return_t benchres2; + adv.mode = BMK_decodeOnly; + benchres2 = BMK_benchMemAdvanced(dstBuffer,dstSize, NULL, 0, &srcSize, 1, 0, &cParams, NULL, 0, ctx, dctx, 0, "File", &adv); + if(benchres2.error) { + return ERROR_RESULT; + } else { + benchres.result.dSpeed = benchres2.result.dSpeed; + } + } + *resultPtr = benchres.result; + return feasible(benchres.result, target); + } else { //feas = 1 or -1 map to 1, 0 respectively + return (feas + 1) >> 1; //relies on INFEASIBLE_RESULT == 0, FEASIBLE_RESULT == 1 + } + } else if (benchres.result.cSize == winnerResult->cSize) { //equal ratio, needs to be better than winner in cSpeed/ dSpeed / cMem + int feas = uncertainFeasibility(uncertaintyConstantC, uncertaintyConstantD, target, &(benchres.result)); + if(feas == 0) { // uncertain feasibility + adv.loopMode = BMK_timeMode; + if(loopDurationC < TIMELOOP_NANOSEC) { + BMK_return_t benchres2; + adv.mode = BMK_compressOnly; + benchres2 = BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, &srcSize, 1, 0, &cParams, NULL, 0, ctx, dctx, 0, "File", &adv); + if(benchres2.error) { + return ERROR_RESULT; + } else { + benchres.result.cSpeed = benchres2.result.cSpeed; + } + } + if(loopDurationD < TIMELOOP_NANOSEC) { + BMK_return_t benchres2; + adv.mode = BMK_decodeOnly; + benchres2 = BMK_benchMemAdvanced(dstBuffer,dstSize, NULL, 0, &srcSize, 1, 0, &cParams, NULL, 0, ctx, dctx, 0, "File", &adv); + if(benchres2.error) { + return ERROR_RESULT; + } else { + benchres.result.dSpeed = benchres2.result.dSpeed; + } + } + + *resultPtr = benchres.result; + return feasible(benchres.result, target) && objective_lt(*winnerResult, benchres.result); + } else if (feas == 1) { //no need to check feasibility compares (maybe only it is chosen as a winner) + int btw = uncertainComparison(uncertaintyConstantC, uncertaintyConstantD, &(benchres.result), winnerResult); + if(btw == -1) { + return INFEASIBLE_RESULT; + } else { //possibly better, benchmark and find out + adv.loopMode = BMK_timeMode; + benchres = BMK_benchMemAdvanced(srcBuffer, srcSize, dstBuffer, dstSize, &srcSize, 1, 0, &cParams, NULL, 0, ctx, dctx, 0, "File", &adv); + *resultPtr = benchres.result; + return objective_lt(*winnerResult, benchres.result); + } + } else { //feas == -1 + return INFEASIBLE_RESULT; //infeasible + } + } else { + return INFEASIBLE_RESULT; //infeasible + } + } else { + return ERROR_RESULT; //BMK error + } + +} +//sameas before, but +/-? +//alternative, just return comparison result, leave caller to worry about feasibility. +//have version of benchMemAdvanced which takes in dstBuffer/cap as well? +//(motivation: repeat tests (maybe just on decompress) don't need further compress runs) +static int infeasibleBench(BMK_result_t* resultPtr, + const void* srcBuffer, size_t srcSize, + void* dstBuffer, size_t dstSize, + ZSTD_CCtx* ctx, ZSTD_DCtx* dctx, + const ZSTD_compressionParameters cParams, + const constraint_t target, + BMK_result_t* winnerResult) { + BMK_advancedParams_t adv = BMK_initAdvancedParams(); + BMK_return_t benchres; + BMK_result_t resultMin, resultMax; + UTIL_time_t startTime; + U64 loopDurationC = 0, loopDurationD = 0; + double uncertaintyConstantC, uncertaintyConstantD; + double winnerRS = resultScore(*winnerResult, srcSize, target); + adv.loopMode = BMK_iterMode; //can only use this for ratio measurement then, super inaccurate timing + adv.nbSeconds = 1; //get ratio and 2x approx speed? //maybe run until twice MIN(minloopinterval * clockDuration) + + (void)startTime; //TODO: actually use this to adjust timing + DISPLAY("WinnerScore: %f\n ", winnerRS); + /* + adv.loopMode = BMK_timeMode; + adv.nbSeconds = 1; */ + benchres = BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, &srcSize, 1, 0, &cParams, NULL, 0, ctx, dctx, 0, "File", &adv); + BMK_printWinner(stdout, CUSTOM_LEVEL, benchres.result, cParams, srcSize); + + if(!benchres.error) { + *resultPtr = benchres.result; + if(eqZero(benchres.result.cSpeed)) { + loopDurationC = 0; + uncertaintyConstantC = 2; + } else { + loopDurationC = ((srcSize * TIMELOOP_NANOSEC) / benchres.result.cSpeed); + //problem - tested in fullbench, saw speed vary 3x between iters, maybe raise uncertaintyConstraint up? + //possibly has to do with initCCtx? or system stuff? + uncertaintyConstantC = MIN((loopDurationC + (double)(2 * g_clockGranularity)/loopDurationC), 2); //.02 seconds + } + + if(eqZero(benchres.result.dSpeed)) { + loopDurationD = 0; + uncertaintyConstantD = 2; + } else { + loopDurationD = ((srcSize * TIMELOOP_NANOSEC) / benchres.result.dSpeed); + //problem - tested in fullbench, saw speed vary 3x between iters, maybe raise uncertaintyConstraint up? + //possibly has to do with initCCtx? or system stuff? + uncertaintyConstantD = MIN((loopDurationD + (double)(2 * g_clockGranularity)/loopDurationD), 2); //.02 seconds + } + + /* benchres's certainty range. */ + resultMax = benchres.result; + resultMin = benchres.result; + resultMax.cSpeed *= uncertaintyConstantC; + resultMax.dSpeed *= uncertaintyConstantD; + resultMin.cSpeed /= uncertaintyConstantC; + resultMin.dSpeed /= uncertaintyConstantD; + (void)resultMin; + //TODO: consider if resultMin is actually needed. + if (winnerRS > resultScore(resultMax, srcSize, target)) { + return INFEASIBLE_RESULT; + } else { + //do this w/o copying / stuff + adv.loopMode = BMK_timeMode; + if(loopDurationC < TIMELOOP_NANOSEC) { + BMK_return_t benchres2; + adv.mode = BMK_compressOnly; + benchres2 = BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, &srcSize, 1, 0, &cParams, NULL, 0, ctx, dctx, 0, "File", &adv); + if(benchres2.error) { + return ERROR_RESULT; + } else { + benchres.result.cSpeed = benchres2.result.cSpeed; + } + } + if(loopDurationD < TIMELOOP_NANOSEC) { + BMK_return_t benchres2; + adv.mode = BMK_decodeOnly; + //TODO: dstBuffer corrupted sometime between top and now + //probably occuring in feasible bench too. + benchres2 = BMK_benchMemAdvanced(dstBuffer, dstSize, NULL, 0, &benchres.result.cSize, 1, 0, &cParams, NULL, 0, ctx, dctx, 0, "File", &adv); + if(benchres2.error) { + return ERROR_RESULT; + } else { + benchres.result.dSpeed = benchres2.result.dSpeed; + } + } + *resultPtr = benchres.result; + return (resultScore(benchres.result, srcSize, target) > winnerRS); + } + + *resultPtr = benchres.result; + } else { + return ERROR_RESULT; //BMK error + } + +} + +/* wrap feasibleBench w/ memotable */ +//TODO: void sanitized and unsanitized ver's so input doesn't double-choose +#define INFEASIBLE_THRESHOLD 200 +static int feasibleBenchMemo(BMK_result_t* resultPtr, + const void* srcBuffer, size_t srcSize, + void* dstBuffer, size_t dstSize, + ZSTD_CCtx* ctx, ZSTD_DCtx* dctx, + const ZSTD_compressionParameters cParams, + const constraint_t target, + BMK_result_t* winnerResult, U8* memoTable, + U32* varyParams, const int varyLen) { + + size_t memind = memoTableInd(&cParams, varyParams, varyLen); + //BMK_translateAdvancedParams(cParams); + if(memoTable[memind] >= INFEASIBLE_THRESHOLD) { + return INFEASIBLE_RESULT; //probably pick a different code for already tested? + //maybe remove this if we incorporate nonrandom location picking? + //what is the intended behavior in this case? + //ignore? stop iterating completely? other? + } else { + int res = feasibleBench(resultPtr, srcBuffer, srcSize, dstBuffer, dstSize, ctx, dctx, + cParams, target, winnerResult); + memoTable[memind] = 255; //tested are all infeasible (other possible values for opti) + return res; + } +} + +//should infeasible stage searching also be memo-marked in the same way? +//don't actually memoize unless result is feasible/error? +static int infeasibleBenchMemo(BMK_result_t* resultPtr, + const void* srcBuffer, size_t srcSize, + void* dstBuffer, size_t dstSize, + ZSTD_CCtx* ctx, ZSTD_DCtx* dctx, + const ZSTD_compressionParameters cParams, + const constraint_t target, + BMK_result_t* winnerResult, U8* memoTable, + U32* varyParams, const int varyLen) { + size_t memind = memoTableInd(&cParams, varyParams, varyLen); + //BMK_translateAdvancedParams(cParams); + if(memoTable[memind] >= INFEASIBLE_THRESHOLD) { + return INFEASIBLE_RESULT; //see feasibleBenchMemo for concerns + } else { + int res = infeasibleBench(resultPtr, srcBuffer, srcSize, dstBuffer, dstSize, ctx, dctx, + cParams, target, winnerResult); + if(res == FEASIBLE_RESULT) { + memoTable[memind] = 255; //infeasible resultscores could still be normal feasible. + } + return res; + } +} + +/* specifically feasibleBenchMemo and infeasibleBenchMemo */ +//maybe not necessary +typedef int (*BMK_benchMemo_t)(BMK_result_t*, const void*, size_t, void*, size_t, ZSTD_CCtx*, ZSTD_DCtx*, + const ZSTD_compressionParameters, const constraint_t, BMK_result_t*, U8*, U32*, const int); + +//varArray should be sanitized when this is called. +//TODO: transition to simpler greedy method if evaluation time is too long? +//would it be better to start at best feasible via feasible or infeasible metric? both? +//possibility climb is infeasible, responsibility of caller to check that. but if something feasible is evaluated, it will be returned +// *actually if it performs too +//sanitize all params here. +//all generation after random should be sanitized. (maybe sanitize random) +//TODO: paramTarget uneeded at this point w/ varArray and init; +static winnerInfo_t climbOnce(constraint_t target, U32* varArray, const int varLen, U8* memoTable, + const void* srcBuffer, size_t srcSize, void* dstBuffer, size_t dstSize, ZSTD_CCtx* ctx, ZSTD_DCtx* dctx, ZSTD_compressionParameters init) { + //pick later initializations non-randomly? high dist from explored nodes. + //how to do this efficiently? (might not be too much of a problem, happens rarely, running time probably dominated by benchmarking) + //distance maximizing selection? + //cparam - currently considered center + //candidate - params to benchmark/results + //winner - best option found so far. + ZSTD_compressionParameters cparam = init; + winnerInfo_t candidateInfo, winnerInfo; + int better = 1; + + winnerInfo.params = init; + winnerInfo.result.cSpeed = 0; + winnerInfo.result.dSpeed = 0; + winnerInfo.result.cMem = (size_t)-1; + winnerInfo.result.cSize = (size_t)-1; + + /* ineasible -> (hopefully) feasible */ + /* when nothing is found, this garbages part 2. */ + { + //TODO: initialize these values! + winnerInfo_t bestFeasible1; /* uses feasibleBench Metric */ + winnerInfo_t bestFeasible2; /* uses resultScore Metric */ + + //init these params + bestFeasible1.params = cparam; + bestFeasible2.params = cparam; + bestFeasible1.result.cSpeed = 0; + bestFeasible1.result.dSpeed = 0; + bestFeasible1.result.cMem = (size_t)-1; + bestFeasible1.result.cSize = (size_t)-1; + bestFeasible2.result.cSpeed = 0; + bestFeasible2.result.dSpeed = 0; + bestFeasible2.result.cMem = (size_t)-1; + bestFeasible2.result.cSize = (size_t)-1; + DISPLAY("Climb Part 1\n"); + while(better) { + + //UTIL_time_t timestart = UTIL_getTime(); TODO: adjust sampling based on time + int i, d; + better = 0; + DISPLAY("Start\n"); + cparam = winnerInfo.params; + BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, srcSize); + candidateInfo.params = cparam; + //all dist-1 targets + for(i = 0; i < varLen; i++) { + paramVaryOnce(varArray[i], 1, &candidateInfo.params); /* +1 */ + candidateInfo.params = sanitizeParams(candidateInfo.params); + //evaluate + //if(!ZSTD_isError(ZSTD_checkCParams(candidateInfo.params))) { + if(cParamValid(candidateInfo.params)) { + int res = infeasibleBenchMemo(&candidateInfo.result, + srcBuffer, srcSize, + dstBuffer, dstSize, + ctx, dctx, + candidateInfo.params, target, &winnerInfo.result, memoTable, + varArray, varLen); + if(res == FEASIBLE_RESULT) { /* synonymous with better when called w/ infeasibleBM */ + winnerInfo = candidateInfo; + //BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, srcSize); + better = 1; + if(feasible(candidateInfo.result, target)) { + bestFeasible2 = winnerInfo; + if(objective_lt(bestFeasible1.result, bestFeasible2.result)) { + bestFeasible1 = bestFeasible2; /* using feasibleBench metric */ + } + } + } + } + candidateInfo.params = cparam; + paramVaryOnce(varArray[i], -1, &candidateInfo.params); /* -1 */ + candidateInfo.params = sanitizeParams(candidateInfo.params); + //evaluate + //if(!ZSTD_isError(ZSTD_checkCParams(candidateInfo.params))) { + if(cParamValid(candidateInfo.params)) { + int res = infeasibleBenchMemo(&candidateInfo.result, + srcBuffer, srcSize, + dstBuffer, dstSize, + ctx, dctx, + candidateInfo.params, target, &winnerInfo.result, memoTable, + varArray, varLen); + if(res == FEASIBLE_RESULT) { + winnerInfo = candidateInfo; + //BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, srcSize); + better = 1; + if(feasible(candidateInfo.result, target)) { + bestFeasible2 = winnerInfo; + if(objective_lt(bestFeasible1.result, bestFeasible2.result)) { + bestFeasible1 = bestFeasible2; + } + } + } + } + } + + if(better) { + continue; + } + //if 'better' enough, skip further parameter search, center there? + //possible improvement - guide direction here w/ knowledge rather than completely random variation. + for(d = 2; d < varLen + 2; d++) { /* varLen is # dimensions */ + for(i = 0; i < 5; i++) { //make ? relative to # of free dimensions. + int res; + candidateInfo.params = cparam; + /* param error checking already done here */ + paramVariation(&candidateInfo.params, varArray, varLen, d); + res = infeasibleBenchMemo(&candidateInfo.result, + srcBuffer, srcSize, + dstBuffer, dstSize, + ctx, dctx, + candidateInfo.params, target, &winnerInfo.result, memoTable, + varArray, varLen); + if(res == FEASIBLE_RESULT) { /* synonymous with better in this case*/ + winnerInfo = candidateInfo; + //BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, srcSize); + better = 1; + if(feasible(candidateInfo.result, target)) { + bestFeasible2 = winnerInfo; + if(objective_lt(bestFeasible1.result, bestFeasible2.result)) { + bestFeasible1 = bestFeasible2; + } + } + } + + } + if(better) { + continue; + } + } + //bias to test previous delta? + //change cparam -> candidate before restart + } + //TODO:Consider if this is best config. idea: explore from obj best keep rbest + cparam = bestFeasible2.params; + candidateInfo = bestFeasible2; + winnerInfo = bestFeasible1; + } + + //is it better to break here instead of bumbling about? + if(winnerInfo.result.cMem == (U32)-1) { + DISPLAY("No Feasible Found\n"); + return winnerInfo; + } + DISPLAY("Climb Part 2\n"); + + better = 1; + /* feasible -> best feasible (hopefully) */ + { + while(better) { + + //UTIL_time_t timestart = UTIL_getTime(); //TODO: if benchmarking is taking too long, be more greedy. + int i, d; + better = 0; + BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, srcSize); + //all dist-1 targets + cparam = winnerInfo.params; //TODO: this messes the taking bestFeasible1, bestFeasible2 + candidateInfo.params = cparam; + for(i = 0; i < varLen; i++) { + paramVaryOnce(varArray[i], 1, &candidateInfo.params); + candidateInfo.params = sanitizeParams(candidateInfo.params); + + //evaluate + //if(!ZSTD_isError(ZSTD_checkCParams(candidateInfo.params))) { + if(cParamValid(candidateInfo.params)) { + int res = feasibleBenchMemo(&candidateInfo.result, + srcBuffer, srcSize, + dstBuffer, dstSize, + ctx, dctx, + candidateInfo.params, target, &winnerInfo.result, memoTable, + varArray, varLen); + if(res == FEASIBLE_RESULT) { + winnerInfo = candidateInfo; + //BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, srcSize); + better = 1; + } + } + candidateInfo.params = cparam; + paramVaryOnce(varArray[i], -1, &candidateInfo.params); + candidateInfo.params = sanitizeParams(candidateInfo.params); + //evaluate + //if(!ZSTD_isError(ZSTD_checkCParams(candidateInfo.params))) { + if(cParamValid(candidateInfo.params)) { + int res = feasibleBenchMemo(&candidateInfo.result, + srcBuffer, srcSize, + dstBuffer, dstSize, + ctx, dctx, + candidateInfo.params, target, &winnerInfo.result, memoTable, + varArray, varLen); + if(res == FEASIBLE_RESULT) { + winnerInfo = candidateInfo; + //BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, srcSize); + better = 1; + } + } + } + //if 'better' enough, skip further parameter search, center there? + //possible improvement - guide direction here w/ knowledge rather than completely random variation. + for(d = 2; d < varLen + 2; d++) { /* varLen is # dimensions */ + for(i = 0; i < 5; i++) { //TODO: make ? relative to # of free dimensions. + int res; + candidateInfo.params = cparam; + /* param error checking already done here */ + paramVariation(&candidateInfo.params, varArray, varLen, d); //info candidateInfo.params is garbage, this is too. + res = feasibleBenchMemo(&candidateInfo.result, + srcBuffer, srcSize, + dstBuffer, dstSize, + ctx, dctx, + candidateInfo.params, target, &winnerInfo.result, memoTable, + varArray, varLen); + if(res == FEASIBLE_RESULT) { + winnerInfo = candidateInfo; + //BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, srcSize); + better = 1; + } + } + if(better) { + continue; + } + } + //bias to test previous delta? + //change cparam -> candidate before restart + } + } + return winnerInfo; +} + +//optimizeForSize but with fixed strategy +//place to configure/filter out strategy specific parameters. +//need args for all buffers and parameter stuff +//sanitization here. + +//flexible parameters: iterations of (failed?) climbing (or if we do non-random, maybe this is when everything is close to visitied) +//weight more on visit for bad results, less on good results/more on later results / ones with more failures. +//allocate memoTable here. +//only real use for paramTarget is to get the fixed values, right? +static winnerInfo_t optimizeFixedStrategy( + const void* srcBuffer, const size_t srcSize, + void* dstBuffer, size_t dstSize, + constraint_t target, ZSTD_compressionParameters paramTarget, + ZSTD_strategy strat, U32* varArray, int varLen) { + int i = 0; //TODO: Temp fix 10 iters, check effects of changing this? + U32* varNew = malloc(sizeof(U32) * varLen); + int varLenNew = sanitizeVarArray(varLen, varArray, varNew, strat); + size_t memoLen = memoTableLen(varNew, varLenNew); + U8* memoTable = malloc(sizeof(U8) * memoLen); + ZSTD_compressionParameters init; + ZSTD_CCtx* ctx = ZSTD_createCCtx(); + ZSTD_DCtx* dctx = ZSTD_createDCtx(); + winnerInfo_t winnerInfo, candidateInfo; + winnerInfo.result.cSpeed = 0; + winnerInfo.result.dSpeed = 0; + winnerInfo.result.cMem = (size_t)(-1); + winnerInfo.result.cSize = (size_t)(-1); + /* so climb is given the right fixed strategy */ + paramTarget.strategy = strat; + /* to pass ZSTD_checkCParams */ + cParamZeroMin(¶mTarget); + memoTableInit(memoTable, paramTarget, target, varNew, varLenNew); + + + init = paramTarget; + + + if(!ctx || !dctx || !memoTable || !varNew) { + DISPLAY("NOT ENOUGH MEMORY ! ! ! \n"); + goto _cleanUp; + } + + while(i < 10) { + DISPLAY("Restart\n"); + randomConstrainedParams(&init, varNew, varLenNew, memoTable); + candidateInfo = climbOnce(target, varNew, varLenNew, memoTable, srcBuffer, srcSize, dstBuffer, dstSize, ctx, dctx, init); + if(objective_lt(winnerInfo.result, candidateInfo.result)) { + winnerInfo = candidateInfo; + DISPLAY("Climb Winner: "); + BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, srcSize); + } + i++; + } + +_cleanUp: + ZSTD_freeCCtx(ctx); + ZSTD_freeDCtx(dctx); + free(memoTable); + free(varNew); + return winnerInfo; +} + +// bigger and (hopefully) better* than optimizeForSize +// TODO: Change level bm'ing to respect constraints. +static int optimizeForSize2(const char* inFileName, constraint_t target, ZSTD_compressionParameters paramTarget) +{ + FILE* const inFile = fopen( inFileName, "rb" ); + U64 const inFileSize = UTIL_getFileSize(inFileName); + size_t benchedSize = BMK_findMaxMem(inFileSize*3) / 3; + void* origBuff; + U32 varArray [NUM_PARAMS]; + int varLen = variableParams(paramTarget, varArray); + /* Init */ + + + if(!cParamValid(paramTarget)) { + return 10; + } + + if (inFile==NULL) { DISPLAY( "Pb opening %s\n", inFileName); return 11; } + if (inFileSize == UTIL_FILESIZE_UNKNOWN) { + DISPLAY("Pb evaluatin size of %s \n", inFileName); + fclose(inFile); + return 11; + } + + /* Memory allocation & restrictions */ + if ((U64)benchedSize > inFileSize) benchedSize = (size_t)inFileSize; + if (benchedSize < inFileSize) { + DISPLAY("Not enough memory for '%s' \n", inFileName); + fclose(inFile); + return 11; + } + + /* Alloc */ + origBuff = malloc(benchedSize); + if(!origBuff) { + DISPLAY("\nError: not enough memory!\n"); + fclose(inFile); + return 12; + } + + /* Fill input buffer */ + DISPLAY("Loading %s... \r", inFileName); + { size_t const readSize = fread(origBuff, 1, benchedSize, inFile); + fclose(inFile); + if(readSize != benchedSize) { + DISPLAY("\nError: problem reading file '%s' !! \n", inFileName); + free(origBuff); + return 13; + } } + + /* bench */ + DISPLAY("\r%79s\r", ""); + DISPLAY("optimizing for %s", inFileName); + if(target.cSpeed != 0) { DISPLAY(" - limit compression speed %u MB/s", target.cSpeed / 1000000); } + if(target.dSpeed != 0) { DISPLAY(" - limit decompression speed %u MB/s", target.dSpeed / 1000000); } + if(target.cMem != (U32)-1) { DISPLAY(" - limit memory %u MB", target.cMem / 1000000); } + DISPLAY("\n"); + findClockGranularity(); + + { ZSTD_CCtx* const ctx = ZSTD_createCCtx(); + ZSTD_DCtx* const dctx = ZSTD_createDCtx(); + winnerInfo_t winner; + //BMK_result_t candidate; + const size_t blockSize = g_blockSize ? g_blockSize : benchedSize; + U32 const maxNbBlocks = (U32) ((benchedSize + (blockSize-1)) / blockSize) + 1; + const size_t maxCompressedSize = ZSTD_compressBound(benchedSize) + (maxNbBlocks * 1024); + void* compressedBuffer = malloc(maxCompressedSize); + + /* init */ + if (ctx==NULL) { DISPLAY("\n ZSTD_createCCtx error \n"); free(origBuff); return 14;} + if(compressedBuffer==NULL) { DISPLAY("\n Allocation Error \n"); free(origBuff); free(ctx); return 15; } + + memset(&winner, 0, sizeof(winner)); + winner.result.cSize = (size_t)(-1); + + + /* find best solution from default params */ + //Can't do this w/ cparameter constraints + //still useful though? + /* + { const int maxSeeds = g_noSeed ? 1 : ZSTD_maxCLevel(); + int i; + for (i=1; i<=maxSeeds; i++) { + ZSTD_compressionParameters const CParams = ZSTD_getCParams(i, blockSize, 0); + BMK_benchParam(&candidate, origBuff, benchedSize, ctx, dctx, CParams); + if (!feasible(candidate, target) ) { + break; + } + if (feasible(candidate,target) && objective_lt(winner.result, candidate)) + { + winner.params = CParams; + winner.result = candidate; + BMK_printWinner(stdout, i, winner.result, winner.params, benchedSize); + } } + }*/ + BMK_printWinner(stdout, CUSTOM_LEVEL, winner.result, winner.params, benchedSize); + + BMK_translateAdvancedParams(winner.params); + + /* start real tests */ + { + if(paramTarget.strategy == 0) { + int st; + for(st = 1; st <= 8; st++) { + winnerInfo_t wc = optimizeFixedStrategy(origBuff, benchedSize, compressedBuffer, maxCompressedSize, + target, paramTarget, st, varArray, varLen); + DISPLAY("StratNum %d\n", st); + if(objective_lt(winner.result, wc.result)) { + winner = wc; + } + } + } else { + winner = optimizeFixedStrategy(origBuff, benchedSize, compressedBuffer, maxCompressedSize, + target, paramTarget, paramTarget.strategy, varArray, varLen); + } + + } + + /* no solution found */ + if(winner.result.cSize == (size_t)-1) { + DISPLAY("No feasible solution found\n"); + return 1; + } + /* end summary */ + BMK_printWinner(stdout, CUSTOM_LEVEL, winner.result, winner.params, benchedSize); + BMK_translateAdvancedParams(winner.params); + DISPLAY("grillParams size - optimizer completed \n"); + + /* clean up*/ + ZSTD_freeCCtx(ctx); + ZSTD_freeDCtx(dctx); + } + + free(origBuff); + return 0; +} + + /* optimizeForSize(): * targetSpeed : expressed in B/s */ -/* if state space is small (from paramTarget) */ +/* expresses targeted compression, decompression speeds and memory requirements */ +/* if state space is small (from paramTarget), exhaustive search? */ +//things to consider : if doing strategy-separate approach, what cutoffs to evaluate each strategy +//or do all? can't be absolute, should be relative after some sort of calibration +//(synthetic? test levels (we don't care about data specifics rn, scale?) ? int optimizeForSize(const char* inFileName, constraint_t target, ZSTD_compressionParameters paramTarget) { FILE* const inFile = fopen( inFileName, "rb" ); @@ -872,7 +1905,7 @@ int optimizeForSize(const char* inFileName, constraint_t target, ZSTD_compressio DISPLAY("optimizing for %s", inFileName); if(target.cSpeed != 0) { DISPLAY(" - limit compression speed %u MB/s", target.cSpeed / 1000000); } if(target.dSpeed != 0) { DISPLAY(" - limit decompression speed %u MB/s", target.dSpeed / 1000000); } - if(target.Mem != 0) { DISPLAY(" - limit memory %u MB", target.Mem / 1000000); } + if(target.cMem != 0) { DISPLAY(" - limit memory %u MB", target.cMem / 1000000); } DISPLAY("\n"); { ZSTD_CCtx* const ctx = ZSTD_createCCtx(); ZSTD_DCtx* const dctx = ZSTD_createDCtx(); @@ -881,12 +1914,13 @@ int optimizeForSize(const char* inFileName, constraint_t target, ZSTD_compressio const size_t blockSize = g_blockSize ? g_blockSize : benchedSize; /* init */ - if (ctx==NULL) { DISPLAY("\n ZSTD_createCCtx error \n"); free(origBuff); return 14;} + if (ctx==NULL) { DISPLAY("\n ZSTD_createCCtx error \n"); free(origBuff); return 14; } + memset(&winner, 0, sizeof(winner)); winner.result.cSize = (size_t)(-1); /* find best solution from default params */ - //Can't do this iteration normally w/ cparameter constraints + //Can't do this w/ cparameter constraints { const int maxSeeds = g_noSeed ? 1 : ZSTD_maxCLevel(); int i; for (i=1; i<=maxSeeds; i++) { @@ -910,15 +1944,17 @@ int optimizeForSize(const char* inFileName, constraint_t target, ZSTD_compressio { time_t const grillStart = time(NULL); do { ZSTD_compressionParameters params = winner.params; - paramVariation(¶ms, paramVarArray, paramCount); + BYTE* b; + paramVariation(¶ms, paramVarArray, paramCount, 4); if ((FUZ_rand(&g_rand) & 31) == 3) params = randomParams(); /* totally random config to improve search space */ params = ZSTD_adjustCParams(params, blockSize, 0); /* exclude faster if already played set of params */ - if (FUZ_rand(&g_rand) & ((1 << NB_TESTS_PLAYED(params))-1)) continue; + if (FUZ_rand(&g_rand) & ((1 << *NB_TESTS_PLAYED(params))-1)) continue; /* test */ - NB_TESTS_PLAYED(params)++; + b = NB_TESTS_PLAYED(params); + (*b)++; BMK_benchParam(&candidate, origBuff, benchedSize, ctx, dctx, params); /* improvement found => new winner */ @@ -1027,7 +2063,8 @@ int main(int argc, const char** argv) U32 optimizer = 0; U32 main_pause = 0; - constraint_t target = { 0 , 0, 0 }; //0 for anything unset + + constraint_t target = { 0, 0, (U32)-1 }; //0 for anything unset ZSTD_compressionParameters paramTarget = { 0, 0, 0, 0, 0, 0, 0 }; assert(argc>=1); /* for exename */ @@ -1053,7 +2090,7 @@ int main(int argc, const char** argv) if (longCommandWArg(&argument, "strategy=") || longCommandWArg(&argument, "strat=")) { paramTarget.strategy = (ZSTD_strategy)(readU32FromChar(&argument)); if (argument[0]==',') { argument++; continue; } else break; } if (longCommandWArg(&argument, "compressionSpeed=") || longCommandWArg(&argument, "cSpeed=")) { target.cSpeed = readU32FromChar(&argument) * 1000000; if (argument[0]==',') { argument++; continue; } else break; } if (longCommandWArg(&argument, "decompressionSpeed=") || longCommandWArg(&argument, "dSpeed=")) { target.dSpeed = readU32FromChar(&argument) * 1000000; if (argument[0]==',') { argument++; continue; } else break; } - if (longCommandWArg(&argument, "compressionMemory=") || longCommandWArg(&argument, "cMem=")) { target.Mem = readU32FromChar(&argument) * 1000000; if (argument[0]==',') { argument++; continue; } else break; } + if (longCommandWArg(&argument, "compressionMemory=") || longCommandWArg(&argument, "cMem=")) { target.cMem = readU32FromChar(&argument) * 1000000; if (argument[0]==',') { argument++; continue; } else break; } /* in MB or MB/s */ DISPLAY("invalid optimization parameter \n"); return 1; @@ -1132,7 +2169,7 @@ int main(int argc, const char** argv) continue; case 'M': argument++; - target.Mem = readU32FromChar(&argument) * 1000000; + target.cMem = readU32FromChar(&argument) * 1000000; continue; case 'w': argument++; @@ -1255,7 +2292,8 @@ int main(int argc, const char** argv) } } else { if (optimizer) { - result = optimizeForSize(input_filename, target, paramTarget); + result = optimizeForSize2(input_filename, target, paramTarget); + //optimizeForSize(input_filename, target, paramTarget); } else { result = benchFiles(argv+filenamesStart, argc-filenamesStart); } } From eb21b7f48224b35ceb91b31a2fe1bb764545724e Mon Sep 17 00:00:00 2001 From: George Lu Date: Mon, 9 Jul 2018 13:44:01 -0700 Subject: [PATCH 112/372] Not crashing --- programs/bench.c | 10 ++++---- tests/paramgrill.c | 59 +++++++++++++++++++++++++++------------------- 2 files changed, 40 insertions(+), 29 deletions(-) diff --git a/programs/bench.c b/programs/bench.c index 0d35f3eb3..5d5b47aba 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -288,7 +288,7 @@ volatile char g_touched; /* takes # of blocks and list of size & stuff for each. */ /* only does looping */ /* note time per loop could be zero if interval too short */ -BMK_customReturn_t __attribute__((optimize("O0"))) BMK_benchFunction( +BMK_customReturn_t BMK_benchFunction( BMK_benchFn_t benchFn, void* benchPayload, BMK_initFn_t initFn, void* initPayload, size_t blockCount, @@ -310,6 +310,7 @@ BMK_customReturn_t __attribute__((optimize("O0"))) BMK_benchFunction( } { + unsigned i, j; for(i = 0; i < blockCount; i++) { for(j = 0; j < srcBlockSizes[i]; j++) { @@ -318,11 +319,11 @@ BMK_customReturn_t __attribute__((optimize("O0"))) BMK_benchFunction( } for(i = 0; i < blockCount; i++) { memset(dstBlockBuffers[i], 0xE5, dstBlockCapacities[i]); /* warm up and erase result buffer */ - //this is written at end proc? where did compressed data get overwritten ny this? } - UTIL_sleepMilli(5); /* give processor time to other processes */ - UTIL_waitForNextTick(); + //UTIL_sleepMilli(5); /* give processor time to other processes */ + //UTIL_waitForNextTick(); + } { @@ -463,7 +464,6 @@ static BMK_return_t BMK_benchMemAdvancedNoAlloc( } { size_t const decodedSize = (size_t)totalDSize64; free(*resultBufferPtr); - //TODO: decodedSize corrupted? *resultBufferPtr = malloc(decodedSize); if (!(*resultBufferPtr)) { EXM_THROW(33, BMK_return_t, "not enough memory"); diff --git a/tests/paramgrill.c b/tests/paramgrill.c index 44bcedef8..77e9f2250 100644 --- a/tests/paramgrill.c +++ b/tests/paramgrill.c @@ -598,8 +598,8 @@ static void paramVariation(ZSTD_compressionParameters* ptr, const U32* varyParam const U32 changeID = FUZ_rand(&g_rand) % (varyLen << 1); paramVaryOnce(varyParams[changeID >> 1], ((changeID & 1) << 1) - 1, &p); } - //validated = !ZSTD_isError(ZSTD_checkCParams(p)); - validated = cParamValid(p); + validated = !ZSTD_isError(ZSTD_checkCParams(p)); + //validated = cParamValid(p); //Make sure memory is at least close to feasible? //ZSTD_estimateCCtxSize thing. @@ -669,23 +669,27 @@ static void memoTableIndInv(ZSTD_compressionParameters* ptr, const U32* varyPara //initializing memoTable /* */ -static void memoTableInit(U8* memoTable, ZSTD_compressionParameters paramConstraints, constraint_t target, const U32* varyParams, const int varyLen) { +static void memoTableInit(U8* memoTable, ZSTD_compressionParameters paramConstraints, constraint_t target, const U32* varyParams, const int varyLen, const size_t srcSize) { size_t i; size_t arrayLen = memoTableLen(varyParams, varyLen); int cwFixed = !paramConstraints.chainLog || !paramConstraints.windowLog; int scFixed = !paramConstraints.searchLog || !paramConstraints.chainLog; + int wFixed = !paramConstraints.windowLog; int j = 0; memset(memoTable, 0, arrayLen); - + cParamZeroMin(¶mConstraints); for(i = 0; i < arrayLen; i++) { memoTableIndInv(¶mConstraints, varyParams, varyLen, i); - BMK_translateAdvancedParams(paramConstraints); if(ZSTD_estimateCCtxSize_usingCParams(paramConstraints) + (1 << paramConstraints.windowLog) > target.cMem) { //infeasible; memoTable[i] = 255; j++; } + //TODO: remove any where memoTable wlog is mark any where windowlog is too big for data. + if(wFixed && (1 << paramConstraints.windowLog) > (srcSize << 1)) { + memoTable[i] = 255; + } /* nil out parameter sets equivalent to others. */ if(cwFixed/* at most least 1 param fixed. */) { if(paramConstraints.strategy == ZSTD_btlazy2 || paramConstraints.strategy == ZSTD_btopt || paramConstraints.strategy == ZSTD_btultra) { @@ -771,8 +775,8 @@ static ZSTD_compressionParameters randomParams(void) p.searchLength=(FUZ_rand(&g_rand) % (ZSTD_SEARCHLENGTH_MAX+1 - ZSTD_SEARCHLENGTH_MIN)) + ZSTD_SEARCHLENGTH_MIN; p.targetLength=(FUZ_rand(&g_rand) % (512)); p.strategy = (ZSTD_strategy) (FUZ_rand(&g_rand) % (ZSTD_btultra +1)); - //validated = !ZSTD_isError(ZSTD_checkCParams(p)); - validated = cParamValid(p); + validated = !ZSTD_isError(ZSTD_checkCParams(p)); + //validated = cParamValid(p); } return p; } @@ -1201,7 +1205,7 @@ static int feasibleBench(BMK_result_t* resultPtr, if(loopDurationD < TIMELOOP_NANOSEC) { BMK_return_t benchres2; adv.mode = BMK_decodeOnly; - benchres2 = BMK_benchMemAdvanced(dstBuffer,dstSize, NULL, 0, &srcSize, 1, 0, &cParams, NULL, 0, ctx, dctx, 0, "File", &adv); + benchres2 = BMK_benchMemAdvanced(dstBuffer,dstSize, NULL, 0, &benchres.result.cSize, 1, 0, &cParams, NULL, 0, ctx, dctx, 0, "File", &adv); if(benchres2.error) { return ERROR_RESULT; } else { @@ -1230,7 +1234,7 @@ static int feasibleBench(BMK_result_t* resultPtr, if(loopDurationD < TIMELOOP_NANOSEC) { BMK_return_t benchres2; adv.mode = BMK_decodeOnly; - benchres2 = BMK_benchMemAdvanced(dstBuffer,dstSize, NULL, 0, &srcSize, 1, 0, &cParams, NULL, 0, ctx, dctx, 0, "File", &adv); + benchres2 = BMK_benchMemAdvanced(dstBuffer,dstSize, NULL, 0, &benchres.result.cSize, 1, 0, &cParams, NULL, 0, ctx, dctx, 0, "File", &adv); if(benchres2.error) { return ERROR_RESULT; } else { @@ -1275,14 +1279,12 @@ static int infeasibleBench(BMK_result_t* resultPtr, BMK_advancedParams_t adv = BMK_initAdvancedParams(); BMK_return_t benchres; BMK_result_t resultMin, resultMax; - UTIL_time_t startTime; U64 loopDurationC = 0, loopDurationD = 0; double uncertaintyConstantC, uncertaintyConstantD; double winnerRS = resultScore(*winnerResult, srcSize, target); adv.loopMode = BMK_iterMode; //can only use this for ratio measurement then, super inaccurate timing adv.nbSeconds = 1; //get ratio and 2x approx speed? //maybe run until twice MIN(minloopinterval * clockDuration) - (void)startTime; //TODO: actually use this to adjust timing DISPLAY("WinnerScore: %f\n ", winnerRS); /* adv.loopMode = BMK_timeMode; @@ -1290,6 +1292,11 @@ static int infeasibleBench(BMK_result_t* resultPtr, benchres = BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, &srcSize, 1, 0, &cParams, NULL, 0, ctx, dctx, 0, "File", &adv); BMK_printWinner(stdout, CUSTOM_LEVEL, benchres.result, cParams, srcSize); + adv.loopMode = BMK_timeMode; + adv.nbSeconds = 1; + benchres = BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, &srcSize, 1, 0, &cParams, NULL, 0, ctx, dctx, 0, "File", &adv); + BMK_printWinner(stdout, CUSTOM_LEVEL, benchres.result, cParams, srcSize); + if(!benchres.error) { *resultPtr = benchres.result; if(eqZero(benchres.result.cSpeed)) { @@ -1372,6 +1379,7 @@ static int feasibleBenchMemo(BMK_result_t* resultPtr, U32* varyParams, const int varyLen) { size_t memind = memoTableInd(&cParams, varyParams, varyLen); + //BMK_translateAdvancedParams(cParams); if(memoTable[memind] >= INFEASIBLE_THRESHOLD) { return INFEASIBLE_RESULT; //probably pick a different code for already tested? @@ -1397,6 +1405,7 @@ static int infeasibleBenchMemo(BMK_result_t* resultPtr, BMK_result_t* winnerResult, U8* memoTable, U32* varyParams, const int varyLen) { size_t memind = memoTableInd(&cParams, varyParams, varyLen); + //BMK_translateAdvancedParams(cParams); if(memoTable[memind] >= INFEASIBLE_THRESHOLD) { return INFEASIBLE_RESULT; //see feasibleBenchMemo for concerns @@ -1422,7 +1431,6 @@ typedef int (*BMK_benchMemo_t)(BMK_result_t*, const void*, size_t, void*, size_t // *actually if it performs too //sanitize all params here. //all generation after random should be sanitized. (maybe sanitize random) -//TODO: paramTarget uneeded at this point w/ varArray and init; static winnerInfo_t climbOnce(constraint_t target, U32* varArray, const int varLen, U8* memoTable, const void* srcBuffer, size_t srcSize, void* dstBuffer, size_t dstSize, ZSTD_CCtx* ctx, ZSTD_DCtx* dctx, ZSTD_compressionParameters init) { //pick later initializations non-randomly? high dist from explored nodes. @@ -1470,12 +1478,13 @@ static winnerInfo_t climbOnce(constraint_t target, U32* varArray, const int varL BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, srcSize); candidateInfo.params = cparam; //all dist-1 targets + //if we early end this, we should also randomize the order these are picked. for(i = 0; i < varLen; i++) { paramVaryOnce(varArray[i], 1, &candidateInfo.params); /* +1 */ candidateInfo.params = sanitizeParams(candidateInfo.params); //evaluate - //if(!ZSTD_isError(ZSTD_checkCParams(candidateInfo.params))) { - if(cParamValid(candidateInfo.params)) { + if(!ZSTD_isError(ZSTD_checkCParams(candidateInfo.params))) { + //if(cParamValid(candidateInfo.params)) { int res = infeasibleBenchMemo(&candidateInfo.result, srcBuffer, srcSize, dstBuffer, dstSize, @@ -1498,8 +1507,8 @@ static winnerInfo_t climbOnce(constraint_t target, U32* varArray, const int varL paramVaryOnce(varArray[i], -1, &candidateInfo.params); /* -1 */ candidateInfo.params = sanitizeParams(candidateInfo.params); //evaluate - //if(!ZSTD_isError(ZSTD_checkCParams(candidateInfo.params))) { - if(cParamValid(candidateInfo.params)) { + if(!ZSTD_isError(ZSTD_checkCParams(candidateInfo.params))) { + //if(cParamValid(candidateInfo.params)) { int res = infeasibleBenchMemo(&candidateInfo.result, srcBuffer, srcSize, dstBuffer, dstSize, @@ -1587,8 +1596,8 @@ static winnerInfo_t climbOnce(constraint_t target, U32* varArray, const int varL candidateInfo.params = sanitizeParams(candidateInfo.params); //evaluate - //if(!ZSTD_isError(ZSTD_checkCParams(candidateInfo.params))) { - if(cParamValid(candidateInfo.params)) { + if(!ZSTD_isError(ZSTD_checkCParams(candidateInfo.params))) { + //if(cParamValid(candidateInfo.params)) { int res = feasibleBenchMemo(&candidateInfo.result, srcBuffer, srcSize, dstBuffer, dstSize, @@ -1605,8 +1614,8 @@ static winnerInfo_t climbOnce(constraint_t target, U32* varArray, const int varL paramVaryOnce(varArray[i], -1, &candidateInfo.params); candidateInfo.params = sanitizeParams(candidateInfo.params); //evaluate - //if(!ZSTD_isError(ZSTD_checkCParams(candidateInfo.params))) { - if(cParamValid(candidateInfo.params)) { + if(!ZSTD_isError(ZSTD_checkCParams(candidateInfo.params))) { + //if(cParamValid(candidateInfo.params)) { int res = feasibleBenchMemo(&candidateInfo.result, srcBuffer, srcSize, dstBuffer, dstSize, @@ -1681,9 +1690,11 @@ static winnerInfo_t optimizeFixedStrategy( /* so climb is given the right fixed strategy */ paramTarget.strategy = strat; /* to pass ZSTD_checkCParams */ - cParamZeroMin(¶mTarget); - memoTableInit(memoTable, paramTarget, target, varNew, varLenNew); + memoTableInit(memoTable, paramTarget, target, varNew, varLenNew, srcSize); + + //needs to happen after memoTableInit as that assumes 0 = undefined. + cParamZeroMin(¶mTarget); init = paramTarget; @@ -1699,7 +1710,7 @@ static winnerInfo_t optimizeFixedStrategy( candidateInfo = climbOnce(target, varNew, varLenNew, memoTable, srcBuffer, srcSize, dstBuffer, dstSize, ctx, dctx, init); if(objective_lt(winnerInfo.result, candidateInfo.result)) { winnerInfo = candidateInfo; - DISPLAY("Climb Winner: "); + DISPLAY("New Winner: "); BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, srcSize); } i++; @@ -1714,7 +1725,7 @@ _cleanUp: } // bigger and (hopefully) better* than optimizeForSize -// TODO: Change level bm'ing to respect constraints. +// TODO: allow accept multiple files like benchFiles or bench.c fn's static int optimizeForSize2(const char* inFileName, constraint_t target, ZSTD_compressionParameters paramTarget) { FILE* const inFile = fopen( inFileName, "rb" ); From fab44388014a8fbb3cf4ba9f46cfdc241807299d Mon Sep 17 00:00:00 2001 From: George Lu Date: Mon, 9 Jul 2018 18:37:54 -0700 Subject: [PATCH 113/372] Dictionary + Multiple file Loading --- tests/paramgrill.c | 209 ++++++++++++++++++++++++++++++++------------- 1 file changed, 151 insertions(+), 58 deletions(-) diff --git a/tests/paramgrill.c b/tests/paramgrill.c index 77e9f2250..2ad8c5752 100644 --- a/tests/paramgrill.c +++ b/tests/paramgrill.c @@ -114,6 +114,7 @@ static U32 g_singleRun = 0; static U32 g_target = 0; static U32 g_noSeed = 0; static ZSTD_compressionParameters g_params = { 0, 0, 0, 0, 0, 0, ZSTD_greedy }; +static UTIL_time_t g_time; /* to be used to compare solution finding speeds to compare to original */ void BMK_SetNbIterations(int nbLoops) { @@ -1141,6 +1142,7 @@ static double calcUncertainty(double speed, size_t srcSize) { static int feasibleBench(BMK_result_t* resultPtr, const void* srcBuffer, size_t srcSize, void* dstBuffer, size_t dstSize, + void* dictBuffer, size_t dictSize, ZSTD_CCtx* ctx, ZSTD_DCtx* dctx, const ZSTD_compressionParameters cParams, const constraint_t target, @@ -1155,7 +1157,7 @@ static int feasibleBench(BMK_result_t* resultPtr, //alternative - test 1 iter for ratio, (possibility of error 3 which is fine), //maybe iter this until 2x measurable for better guarantee? DISPLAY("Feas:\n"); - benchres = BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, &srcSize, 1, 0, &cParams, NULL, 0, ctx, dctx, 0, "File", &adv); + benchres = BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, &srcSize, 1, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); if(benchres.error) { DISPLAY("ERROR %d !!\n", benchres.error); } @@ -1195,7 +1197,7 @@ static int feasibleBench(BMK_result_t* resultPtr, if(loopDurationC < TIMELOOP_NANOSEC) { BMK_return_t benchres2; adv.mode = BMK_compressOnly; - benchres2 = BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, &srcSize, 1, 0, &cParams, NULL, 0, ctx, dctx, 0, "File", &adv); + benchres2 = BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, &srcSize, 1, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); if(benchres2.error) { return ERROR_RESULT; } else { @@ -1205,7 +1207,7 @@ static int feasibleBench(BMK_result_t* resultPtr, if(loopDurationD < TIMELOOP_NANOSEC) { BMK_return_t benchres2; adv.mode = BMK_decodeOnly; - benchres2 = BMK_benchMemAdvanced(dstBuffer,dstSize, NULL, 0, &benchres.result.cSize, 1, 0, &cParams, NULL, 0, ctx, dctx, 0, "File", &adv); + benchres2 = BMK_benchMemAdvanced(dstBuffer,dstSize, NULL, 0, &benchres.result.cSize, 1, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); if(benchres2.error) { return ERROR_RESULT; } else { @@ -1224,7 +1226,7 @@ static int feasibleBench(BMK_result_t* resultPtr, if(loopDurationC < TIMELOOP_NANOSEC) { BMK_return_t benchres2; adv.mode = BMK_compressOnly; - benchres2 = BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, &srcSize, 1, 0, &cParams, NULL, 0, ctx, dctx, 0, "File", &adv); + benchres2 = BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, &srcSize, 1, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); if(benchres2.error) { return ERROR_RESULT; } else { @@ -1234,7 +1236,7 @@ static int feasibleBench(BMK_result_t* resultPtr, if(loopDurationD < TIMELOOP_NANOSEC) { BMK_return_t benchres2; adv.mode = BMK_decodeOnly; - benchres2 = BMK_benchMemAdvanced(dstBuffer,dstSize, NULL, 0, &benchres.result.cSize, 1, 0, &cParams, NULL, 0, ctx, dctx, 0, "File", &adv); + benchres2 = BMK_benchMemAdvanced(dstBuffer,dstSize, NULL, 0, &benchres.result.cSize, 1, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); if(benchres2.error) { return ERROR_RESULT; } else { @@ -1250,7 +1252,7 @@ static int feasibleBench(BMK_result_t* resultPtr, return INFEASIBLE_RESULT; } else { //possibly better, benchmark and find out adv.loopMode = BMK_timeMode; - benchres = BMK_benchMemAdvanced(srcBuffer, srcSize, dstBuffer, dstSize, &srcSize, 1, 0, &cParams, NULL, 0, ctx, dctx, 0, "File", &adv); + benchres = BMK_benchMemAdvanced(srcBuffer, srcSize, dstBuffer, dstSize, &srcSize, 1, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); *resultPtr = benchres.result; return objective_lt(*winnerResult, benchres.result); } @@ -1272,6 +1274,7 @@ static int feasibleBench(BMK_result_t* resultPtr, static int infeasibleBench(BMK_result_t* resultPtr, const void* srcBuffer, size_t srcSize, void* dstBuffer, size_t dstSize, + void* dictBuffer, size_t dictSize, ZSTD_CCtx* ctx, ZSTD_DCtx* dctx, const ZSTD_compressionParameters cParams, const constraint_t target, @@ -1289,12 +1292,14 @@ static int infeasibleBench(BMK_result_t* resultPtr, /* adv.loopMode = BMK_timeMode; adv.nbSeconds = 1; */ - benchres = BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, &srcSize, 1, 0, &cParams, NULL, 0, ctx, dctx, 0, "File", &adv); + benchres = BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, &srcSize, 1, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); + BMK_printWinner(stdout, CUSTOM_LEVEL, benchres.result, cParams, srcSize); + benchres = BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, &srcSize, 1, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); BMK_printWinner(stdout, CUSTOM_LEVEL, benchres.result, cParams, srcSize); adv.loopMode = BMK_timeMode; adv.nbSeconds = 1; - benchres = BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, &srcSize, 1, 0, &cParams, NULL, 0, ctx, dctx, 0, "File", &adv); + benchres = BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, &srcSize, 1, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); BMK_printWinner(stdout, CUSTOM_LEVEL, benchres.result, cParams, srcSize); if(!benchres.error) { @@ -1336,7 +1341,7 @@ static int infeasibleBench(BMK_result_t* resultPtr, if(loopDurationC < TIMELOOP_NANOSEC) { BMK_return_t benchres2; adv.mode = BMK_compressOnly; - benchres2 = BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, &srcSize, 1, 0, &cParams, NULL, 0, ctx, dctx, 0, "File", &adv); + benchres2 = BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, &srcSize, 1, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); if(benchres2.error) { return ERROR_RESULT; } else { @@ -1348,7 +1353,7 @@ static int infeasibleBench(BMK_result_t* resultPtr, adv.mode = BMK_decodeOnly; //TODO: dstBuffer corrupted sometime between top and now //probably occuring in feasible bench too. - benchres2 = BMK_benchMemAdvanced(dstBuffer, dstSize, NULL, 0, &benchres.result.cSize, 1, 0, &cParams, NULL, 0, ctx, dctx, 0, "File", &adv); + benchres2 = BMK_benchMemAdvanced(dstBuffer, dstSize, NULL, 0, &benchres.result.cSize, 1, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); if(benchres2.error) { return ERROR_RESULT; } else { @@ -1372,6 +1377,7 @@ static int infeasibleBench(BMK_result_t* resultPtr, static int feasibleBenchMemo(BMK_result_t* resultPtr, const void* srcBuffer, size_t srcSize, void* dstBuffer, size_t dstSize, + void* dictBuffer, size_t dictSize, ZSTD_CCtx* ctx, ZSTD_DCtx* dctx, const ZSTD_compressionParameters cParams, const constraint_t target, @@ -1387,7 +1393,7 @@ static int feasibleBenchMemo(BMK_result_t* resultPtr, //what is the intended behavior in this case? //ignore? stop iterating completely? other? } else { - int res = feasibleBench(resultPtr, srcBuffer, srcSize, dstBuffer, dstSize, ctx, dctx, + int res = feasibleBench(resultPtr, srcBuffer, srcSize, dstBuffer, dstSize, dictBuffer, dictSize, ctx, dctx, cParams, target, winnerResult); memoTable[memind] = 255; //tested are all infeasible (other possible values for opti) return res; @@ -1399,6 +1405,7 @@ static int feasibleBenchMemo(BMK_result_t* resultPtr, static int infeasibleBenchMemo(BMK_result_t* resultPtr, const void* srcBuffer, size_t srcSize, void* dstBuffer, size_t dstSize, + void* dictBuffer, size_t dictSize, ZSTD_CCtx* ctx, ZSTD_DCtx* dctx, const ZSTD_compressionParameters cParams, const constraint_t target, @@ -1410,7 +1417,7 @@ static int infeasibleBenchMemo(BMK_result_t* resultPtr, if(memoTable[memind] >= INFEASIBLE_THRESHOLD) { return INFEASIBLE_RESULT; //see feasibleBenchMemo for concerns } else { - int res = infeasibleBench(resultPtr, srcBuffer, srcSize, dstBuffer, dstSize, ctx, dctx, + int res = infeasibleBench(resultPtr, srcBuffer, srcSize, dstBuffer, dstSize, dictBuffer, dictSize, ctx, dctx, cParams, target, winnerResult); if(res == FEASIBLE_RESULT) { memoTable[memind] = 255; //infeasible resultscores could still be normal feasible. @@ -1432,7 +1439,7 @@ typedef int (*BMK_benchMemo_t)(BMK_result_t*, const void*, size_t, void*, size_t //sanitize all params here. //all generation after random should be sanitized. (maybe sanitize random) static winnerInfo_t climbOnce(constraint_t target, U32* varArray, const int varLen, U8* memoTable, - const void* srcBuffer, size_t srcSize, void* dstBuffer, size_t dstSize, ZSTD_CCtx* ctx, ZSTD_DCtx* dctx, ZSTD_compressionParameters init) { + const void* srcBuffer, size_t srcSize, void* dstBuffer, size_t dstSize, void* dictBuffer, size_t dictSize, ZSTD_CCtx* ctx, ZSTD_DCtx* dctx, ZSTD_compressionParameters init) { //pick later initializations non-randomly? high dist from explored nodes. //how to do this efficiently? (might not be too much of a problem, happens rarely, running time probably dominated by benchmarking) //distance maximizing selection? @@ -1488,6 +1495,7 @@ static winnerInfo_t climbOnce(constraint_t target, U32* varArray, const int varL int res = infeasibleBenchMemo(&candidateInfo.result, srcBuffer, srcSize, dstBuffer, dstSize, + dictBuffer, dictSize, ctx, dctx, candidateInfo.params, target, &winnerInfo.result, memoTable, varArray, varLen); @@ -1512,6 +1520,7 @@ static winnerInfo_t climbOnce(constraint_t target, U32* varArray, const int varL int res = infeasibleBenchMemo(&candidateInfo.result, srcBuffer, srcSize, dstBuffer, dstSize, + dictBuffer, dictSize, ctx, dctx, candidateInfo.params, target, &winnerInfo.result, memoTable, varArray, varLen); @@ -1519,7 +1528,7 @@ static winnerInfo_t climbOnce(constraint_t target, U32* varArray, const int varL winnerInfo = candidateInfo; //BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, srcSize); better = 1; - if(feasible(candidateInfo.result, target)) { + if(feasible(candidateInfo.result, target)) { //TODO: maybe just break here and move on to part 2? bestFeasible2 = winnerInfo; if(objective_lt(bestFeasible1.result, bestFeasible2.result)) { bestFeasible1 = bestFeasible2; @@ -1535,14 +1544,15 @@ static winnerInfo_t climbOnce(constraint_t target, U32* varArray, const int varL //if 'better' enough, skip further parameter search, center there? //possible improvement - guide direction here w/ knowledge rather than completely random variation. for(d = 2; d < varLen + 2; d++) { /* varLen is # dimensions */ - for(i = 0; i < 5; i++) { //make ? relative to # of free dimensions. + for(i = 0; i < 2 * varLen + 2; i++) { int res; candidateInfo.params = cparam; /* param error checking already done here */ paramVariation(&candidateInfo.params, varArray, varLen, d); res = infeasibleBenchMemo(&candidateInfo.result, srcBuffer, srcSize, - dstBuffer, dstSize, + dstBuffer, dstSize, + dictBuffer, dictSize, ctx, dctx, candidateInfo.params, target, &winnerInfo.result, memoTable, varArray, varLen); @@ -1601,6 +1611,7 @@ static winnerInfo_t climbOnce(constraint_t target, U32* varArray, const int varL int res = feasibleBenchMemo(&candidateInfo.result, srcBuffer, srcSize, dstBuffer, dstSize, + dictBuffer, dictSize, ctx, dctx, candidateInfo.params, target, &winnerInfo.result, memoTable, varArray, varLen); @@ -1619,6 +1630,7 @@ static winnerInfo_t climbOnce(constraint_t target, U32* varArray, const int varL int res = feasibleBenchMemo(&candidateInfo.result, srcBuffer, srcSize, dstBuffer, dstSize, + dictBuffer, dictSize, ctx, dctx, candidateInfo.params, target, &winnerInfo.result, memoTable, varArray, varLen); @@ -1632,7 +1644,7 @@ static winnerInfo_t climbOnce(constraint_t target, U32* varArray, const int varL //if 'better' enough, skip further parameter search, center there? //possible improvement - guide direction here w/ knowledge rather than completely random variation. for(d = 2; d < varLen + 2; d++) { /* varLen is # dimensions */ - for(i = 0; i < 5; i++) { //TODO: make ? relative to # of free dimensions. + for(i = 0; i < 2 * varLen + 2; i++) { int res; candidateInfo.params = cparam; /* param error checking already done here */ @@ -1640,6 +1652,7 @@ static winnerInfo_t climbOnce(constraint_t target, U32* varArray, const int varL res = feasibleBenchMemo(&candidateInfo.result, srcBuffer, srcSize, dstBuffer, dstSize, + dictBuffer, dictSize, ctx, dctx, candidateInfo.params, target, &winnerInfo.result, memoTable, varArray, varLen); @@ -1672,6 +1685,7 @@ static winnerInfo_t climbOnce(constraint_t target, U32* varArray, const int varL static winnerInfo_t optimizeFixedStrategy( const void* srcBuffer, const size_t srcSize, void* dstBuffer, size_t dstSize, + void* dictBuffer, size_t dictSize, constraint_t target, ZSTD_compressionParameters paramTarget, ZSTD_strategy strat, U32* varArray, int varLen) { int i = 0; //TODO: Temp fix 10 iters, check effects of changing this? @@ -1706,12 +1720,14 @@ static winnerInfo_t optimizeFixedStrategy( while(i < 10) { DISPLAY("Restart\n"); + //TODO: look into improving this to maximize distance from searched infeasible stuff / towards promising regions? randomConstrainedParams(&init, varNew, varLenNew, memoTable); - candidateInfo = climbOnce(target, varNew, varLenNew, memoTable, srcBuffer, srcSize, dstBuffer, dstSize, ctx, dctx, init); + candidateInfo = climbOnce(target, varNew, varLenNew, memoTable, srcBuffer, srcSize, dstBuffer, dstSize, dictBuffer, dictSize, ctx, dctx, init); if(objective_lt(winnerInfo.result, candidateInfo.result)) { winnerInfo = candidateInfo; DISPLAY("New Winner: "); BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, srcSize); + i = 0; } i++; } @@ -1724,59 +1740,125 @@ _cleanUp: return winnerInfo; } +static int BMK_loadFiles(void* buffer, size_t bufferSize, + size_t* fileSizes, const char* const * const fileNamesTable, + unsigned nbFiles) +{ + size_t pos = 0, totalSize = 0; + unsigned n; + for (n=0; n bufferSize-pos) fileSize = bufferSize-pos, nbFiles=n; /* buffer too small - stop after this file */ + { size_t const readSize = fread(((char*)buffer)+pos, 1, (size_t)fileSize, f); + if (readSize != (size_t)fileSize) { + DISPLAY("could not read %s", fileNamesTable[n]); + return 11; + } + pos += readSize; } + fileSizes[n] = (size_t)fileSize; + totalSize += (size_t)fileSize; + fclose(f); + } + + if (totalSize == 0) { DISPLAY("no data to bench\n"); return 12; } + return 0; +} + // bigger and (hopefully) better* than optimizeForSize // TODO: allow accept multiple files like benchFiles or bench.c fn's -static int optimizeForSize2(const char* inFileName, constraint_t target, ZSTD_compressionParameters paramTarget) +static int optimizeForSize2(const char* const * const fileNamesTable, const size_t nbFiles, const char* dictFileName, constraint_t target, ZSTD_compressionParameters paramTarget) { - FILE* const inFile = fopen( inFileName, "rb" ); - U64 const inFileSize = UTIL_getFileSize(inFileName); - size_t benchedSize = BMK_findMaxMem(inFileSize*3) / 3; + size_t benchedSize; void* origBuff; + void* dictBuffer; + size_t dictBufferSize; U32 varArray [NUM_PARAMS]; int varLen = variableParams(paramTarget, varArray); /* Init */ - - if(!cParamValid(paramTarget)) { return 10; } - if (inFile==NULL) { DISPLAY( "Pb opening %s\n", inFileName); return 11; } - if (inFileSize == UTIL_FILESIZE_UNKNOWN) { - DISPLAY("Pb evaluatin size of %s \n", inFileName); - fclose(inFile); - return 11; - } - - /* Memory allocation & restrictions */ - if ((U64)benchedSize > inFileSize) benchedSize = (size_t)inFileSize; - if (benchedSize < inFileSize) { - DISPLAY("Not enough memory for '%s' \n", inFileName); - fclose(inFile); - return 11; - } - - /* Alloc */ - origBuff = malloc(benchedSize); - if(!origBuff) { - DISPLAY("\nError: not enough memory!\n"); - fclose(inFile); - return 12; + /* load dictionary*/ + if (dictFileName != NULL) { + U64 const dictFileSize = UTIL_getFileSize(dictFileName); + if (dictFileSize > 64 MB) { + DISPLAY("dictionary file %s too large", dictFileName); + return 10; + } + dictBufferSize = (size_t)dictFileSize; + dictBuffer = malloc(dictBufferSize); + if (dictBuffer==NULL) { + DISPLAY("not enough memory for dictionary (%u bytes)", + (U32)dictBufferSize); + return 11; + } + { + int errorCode = BMK_loadFiles(dictBuffer, dictBufferSize, &dictBufferSize, &dictFileName, 1); + if(errorCode) { + free(dictBuffer); + return errorCode; + } + } } /* Fill input buffer */ - DISPLAY("Loading %s... \r", inFileName); - { size_t const readSize = fread(origBuff, 1, benchedSize, inFile); - fclose(inFile); - if(readSize != benchedSize) { - DISPLAY("\nError: problem reading file '%s' !! \n", inFileName); + if(nbFiles == 1) { + DISPLAY("Loading %s... \r", fileNamesTable[0]); + } else { + DISPLAY("Loading %zd Files... \r", nbFiles); + } + + { + U64 const totalSizeToLoad = UTIL_getTotalFileSize(fileNamesTable, nbFiles); + int ec; + size_t* fileSizes = calloc(sizeof(size_t),nbFiles); + benchedSize = BMK_findMaxMem(totalSizeToLoad * 3) / 3; + origBuff = malloc(benchedSize); + if(!origBuff || !fileSizes) { + DISPLAY("Not enough memory for stuff\n"); free(origBuff); - return 13; - } } + free(fileSizes); + free(dictBuffer); + return 1; + } + ec = BMK_loadFiles(origBuff, benchedSize, fileSizes, fileNamesTable, nbFiles); + if(ec) { + DISPLAY("Error Loading Files"); + free(origBuff); + free(fileSizes); + free(dictBuffer); + return ec; + } + free(fileSizes); + } + + /* bench */ DISPLAY("\r%79s\r", ""); - DISPLAY("optimizing for %s", inFileName); + if(nbFiles == 1) { + DISPLAY("optimizing for %s", fileNamesTable[0]); + } else { + DISPLAY("optimizing for %zd Files", nbFiles); + } if(target.cSpeed != 0) { DISPLAY(" - limit compression speed %u MB/s", target.cSpeed / 1000000); } if(target.dSpeed != 0) { DISPLAY(" - limit decompression speed %u MB/s", target.dSpeed / 1000000); } if(target.cMem != (U32)-1) { DISPLAY(" - limit memory %u MB", target.cMem / 1000000); } @@ -1828,7 +1910,7 @@ static int optimizeForSize2(const char* inFileName, constraint_t target, ZSTD_co if(paramTarget.strategy == 0) { int st; for(st = 1; st <= 8; st++) { - winnerInfo_t wc = optimizeFixedStrategy(origBuff, benchedSize, compressedBuffer, maxCompressedSize, + winnerInfo_t wc = optimizeFixedStrategy(origBuff, benchedSize, compressedBuffer, maxCompressedSize, dictBuffer, dictBufferSize, target, paramTarget, st, varArray, varLen); DISPLAY("StratNum %d\n", st); if(objective_lt(winner.result, wc.result)) { @@ -1836,7 +1918,7 @@ static int optimizeForSize2(const char* inFileName, constraint_t target, ZSTD_co } } } else { - winner = optimizeFixedStrategy(origBuff, benchedSize, compressedBuffer, maxCompressedSize, + winner = optimizeFixedStrategy(origBuff, benchedSize, compressedBuffer, maxCompressedSize, dictBuffer, dictBufferSize, target, paramTarget, paramTarget.strategy, varArray, varLen); } @@ -2070,7 +2152,8 @@ int main(int argc, const char** argv) filenamesStart=0, result; const char* exename=argv[0]; - const char* input_filename=0; + const char* input_filename = 0; + const char* dictFileName = 0; U32 optimizer = 0; U32 main_pause = 0; @@ -2283,6 +2366,17 @@ int main(int argc, const char** argv) g_grillDuration_s = (double)readU32FromChar(&argument); break; + /* load dictionary file (only applicable for optimizer rn) */ + case 'D': + if(i == argc - 1) { //last argument, return error. + DISPLAY("Dictionary file expected but not given\n"); + return 1; + } else { + i++; + dictFileName = argv[i]; + } + break; + /* Unknown command */ default : return badusage(exename); } @@ -2303,8 +2397,7 @@ int main(int argc, const char** argv) } } else { if (optimizer) { - result = optimizeForSize2(input_filename, target, paramTarget); - //optimizeForSize(input_filename, target, paramTarget); + result = optimizeForSize2(argv+filenamesStart, argc-filenamesStart, dictFileName, target, paramTarget); } else { result = benchFiles(argv+filenamesStart, argc-filenamesStart); } } From 3adc217ea47f93f11e32ca6d3bf2f103ff3dbbd0 Mon Sep 17 00:00:00 2001 From: George Lu Date: Thu, 12 Jul 2018 17:30:39 -0700 Subject: [PATCH 114/372] Total Changes: Add different constraint types (decompression speed, compression memory, parameter constraints) Separate search space by strategy + strategy selection Memoize results Real random restarts Support multiple files Support Dictionary inputs Debug Macro for extra printing --- programs/bench.c | 14 +- programs/bench.h | 4 +- tests/Makefile | 2 +- tests/fullbench.c | 4 +- tests/paramgrill.c | 1090 ++++++++++++++++++++++---------------------- 5 files changed, 548 insertions(+), 566 deletions(-) diff --git a/programs/bench.c b/programs/bench.c index 5d5b47aba..f5184fa8f 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -85,7 +85,7 @@ static UTIL_time_t g_displayClock = UTIL_TIME_INITIALIZER; * Exceptions ***************************************/ #ifndef DEBUG -# define DEBUG 1 +# define DEBUG 0 #endif #define DEBUGOUTPUT(...) { if (DEBUG) DISPLAY(__VA_ARGS__); } @@ -188,7 +188,7 @@ static void BMK_initCCtx(ZSTD_CCtx* ctx, ZSTD_CCtx_setParameter(ctx, ZSTD_p_searchLog, comprParams->searchLog); ZSTD_CCtx_setParameter(ctx, ZSTD_p_minMatch, comprParams->searchLength); ZSTD_CCtx_setParameter(ctx, ZSTD_p_targetLength, comprParams->targetLength); - ZSTD_CCtx_setParameter(ctx, ZSTD_p_compressionStrategy, comprParams->strategy); + ZSTD_CCtx_setParameter(ctx, ZSTD_p_compressionStrategy , comprParams->strategy); ZSTD_CCtx_loadDictionary(ctx, dictBuffer, dictBufferSize); } @@ -293,7 +293,7 @@ BMK_customReturn_t BMK_benchFunction( BMK_initFn_t initFn, void* initPayload, size_t blockCount, const void* const * const srcBlockBuffers, const size_t* srcBlockSizes, - void* const * const dstBlockBuffers, const size_t* dstBlockCapacities, + void** const dstBlockBuffers, size_t* dstBlockCapacities, unsigned nbLoops) { size_t srcSize = 0, dstSize = 0, ind = 0; U64 totalTime; @@ -338,6 +338,11 @@ BMK_customReturn_t BMK_benchFunction( j, (U32)dstBlockCapacities[j], ZSTD_getErrorName(res)); } else if(firstIter) { dstSize += res; + //Make compressed blocks continuous + if(j != blockCount - 1) { + dstBlockBuffers[j+1] = (void*)((char*)dstBlockBuffers[j] + res); + dstBlockCapacities[j] = res; + } } } firstIter = 0; @@ -370,13 +375,14 @@ void BMK_freeTimeState(BMK_timedFnState_t* state) { free(state); } +/* make option for dstBlocks to be */ BMK_customTimedReturn_t BMK_benchFunctionTimed( BMK_timedFnState_t* cont, BMK_benchFn_t benchFn, void* benchPayload, BMK_initFn_t initFn, void* initPayload, size_t blockCount, const void* const* const srcBlockBuffers, const size_t* srcBlockSizes, - void* const* const dstBlockBuffers, const size_t* dstBlockCapacities) + void** const dstBlockBuffers, size_t* dstBlockCapacities) { U64 fastest = cont->fastestTime; int completed = 0; diff --git a/programs/bench.h b/programs/bench.h index 625b65757..1a298cc48 100644 --- a/programs/bench.h +++ b/programs/bench.h @@ -191,7 +191,7 @@ BMK_customReturn_t BMK_benchFunction( BMK_initFn_t initFn, void* initPayload, size_t blockCount, const void* const * const srcBuffers, const size_t* srcSizes, - void* const * const dstBuffers, const size_t* dstCapacities, + void** const dstBuffers, size_t* dstCapacities, unsigned nbLoops); @@ -220,7 +220,7 @@ BMK_customTimedReturn_t BMK_benchFunctionTimed(BMK_timedFnState_t* cont, BMK_initFn_t initFn, void* initPayload, size_t blockCount, const void* const * const srcBlockBuffers, const size_t* srcBlockSizes, - void* const * const dstBlockBuffers, const size_t* dstBlockCapacities); + void** const dstBlockBuffers, size_t* dstBlockCapacities); #endif /* BENCH_H_121279284357 */ diff --git a/tests/Makefile b/tests/Makefile index ecf7fe2a7..81e685780 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -200,7 +200,7 @@ zstreamtest-dll : $(ZSTDDIR)/common/xxhash.c # xxh symbols not exposed from dll zstreamtest-dll : $(ZSTREAM_LOCAL_FILES) $(CC) $(CPPFLAGS) $(CFLAGS) $(filter %.c,$^) $(LDFLAGS) -o $@$(EXT) -#paramgrill : DEBUGFLAGS = # turn off assert() for speed measurements +paramgrill : DEBUGFLAGS = # turn off assert() for speed measurements paramgrill : $(ZSTD_FILES) $(PRGDIR)/bench.c $(PRGDIR)/datagen.c paramgrill.c $(CC) $(FLAGS) $^ -lm -o $@$(EXT) diff --git a/tests/fullbench.c b/tests/fullbench.c index 7859745a3..9e7639f92 100644 --- a/tests/fullbench.c +++ b/tests/fullbench.c @@ -336,7 +336,7 @@ size_t local_ZSTD_decompressContinue(const void* src, size_t srcSize, void* dst, static size_t benchMem(const void* src, size_t srcSize, U32 benchNb, int cLevel, ZSTD_compressionParameters* cparams) { BYTE* dstBuff; - size_t const dstBuffSize = ZSTD_compressBound(srcSize); + size_t dstBuffSize = ZSTD_compressBound(srcSize); void* buff2, *buff1; const char* benchName; BMK_benchFn_t benchFunction; @@ -516,7 +516,7 @@ static size_t benchMem(const void* src, size_t srcSize, U32 benchNb, int cLevel, { r = BMK_benchFunction(benchFunction, buff2, NULL, NULL, 1, &src, &srcSize, - (void * const * const)&dstBuff, &dstBuffSize, g_nbIterations); + (void **)&dstBuff, &dstBuffSize, g_nbIterations); if(r.error) { DISPLAY("ERROR %d ! ! \n", r.error); errorcode = r.error; diff --git a/tests/paramgrill.c b/tests/paramgrill.c index 2ad8c5752..9af60a740 100644 --- a/tests/paramgrill.c +++ b/tests/paramgrill.c @@ -58,6 +58,11 @@ static const int g_maxNbVariations = 64; * Macros **************************************/ #define DISPLAY(...) fprintf(stderr, __VA_ARGS__) +#define TIMED 0 +#ifndef DEBUG +# define DEBUG 0 +#endif +#define DEBUGOUTPUT(...) { if (DEBUG) DISPLAY(__VA_ARGS__); } #undef MIN #undef MAX @@ -81,8 +86,8 @@ static const int g_maxNbVariations = 64; #define ZSTD_WINDOWLOG_MAX 27 //no long range stuff for now. //make 2^[0,10] w/ 999 -#define ZSTD_TARGETLENGTH_MIN 0 //actually targeLengthlog min -#define ZSTD_TARGETLENGTH_MAX 10 +#define ZSTD_TARGETLENGTH_MIN 0 +#define ZSTD_TARGETLENGTH_MAX 999 //#define ZSTD_TARGETLENGTH_MAX 1024 #define WLOG_RANGE (ZSTD_WINDOWLOG_MAX - ZSTD_WINDOWLOG_MIN + 1) @@ -90,15 +95,14 @@ static const int g_maxNbVariations = 64; #define HLOG_RANGE (ZSTD_HASHLOG_MAX - ZSTD_HASHLOG_MIN + 1) #define SLOG_RANGE (ZSTD_SEARCHLOG_MAX - ZSTD_SEARCHLOG_MIN + 1) #define SLEN_RANGE (ZSTD_SEARCHLENGTH_MAX - ZSTD_SEARCHLENGTH_MIN + 1) -#define TLEN_RANGE 11 +#define TLEN_RANGE 12 +//TLEN_RANGE = 0, 2^0 to 2^10; //hard coded since we only use powers of 2 (and 999 ~ 1024) -static const int mintable[NUM_PARAMS] = { ZSTD_WINDOWLOG_MIN, ZSTD_CHAINLOG_MIN, ZSTD_HASHLOG_MIN, ZSTD_SEARCHLOG_MIN, ZSTD_SEARCHLENGTH_MIN, ZSTD_TARGETLENGTH_MIN }; -static const int maxtable[NUM_PARAMS] = { ZSTD_WINDOWLOG_MAX, ZSTD_CHAINLOG_MAX, ZSTD_HASHLOG_MAX, ZSTD_SEARCHLOG_MAX, ZSTD_SEARCHLENGTH_MAX, ZSTD_TARGETLENGTH_MAX }; +//static const int mintable[NUM_PARAMS] = { ZSTD_WINDOWLOG_MIN, ZSTD_CHAINLOG_MIN, ZSTD_HASHLOG_MIN, ZSTD_SEARCHLOG_MIN, ZSTD_SEARCHLENGTH_MIN, ZSTD_TARGETLENGTH_MIN }; +//static const int maxtable[NUM_PARAMS] = { ZSTD_WINDOWLOG_MAX, ZSTD_CHAINLOG_MAX, ZSTD_HASHLOG_MAX, ZSTD_SEARCHLOG_MAX, ZSTD_SEARCHLENGTH_MAX, ZSTD_TARGETLENGTH_MAX }; static const int rangetable[NUM_PARAMS] = { WLOG_RANGE, CLOG_RANGE, HLOG_RANGE, SLOG_RANGE, SLEN_RANGE, TLEN_RANGE }; -//use grid-search or something when space is small enough? -#define SMALL_SEARCH_SPACE 1000 /*-************************************ * Benchmark Parameters **************************************/ @@ -201,7 +205,7 @@ static void findClockGranularity(void) { } } } while(i < 10); - DISPLAY("Granularity: %llu\n", (unsigned long long)g_clockGranularity); + DEBUGOUTPUT("Granularity: %llu\n", (unsigned long long)g_clockGranularity); } typedef struct { @@ -225,6 +229,10 @@ static int cParamValid(ZSTD_compressionParameters paramTarget) { CLAMPCHECK(paramTarget.searchLength, ZSTD_SEARCHLENGTH_MIN, ZSTD_SEARCHLENGTH_MAX); CLAMPCHECK(paramTarget.windowLog, ZSTD_WINDOWLOG_MIN, ZSTD_WINDOWLOG_MAX); CLAMPCHECK(paramTarget.chainLog, ZSTD_CHAINLOG_MIN, ZSTD_CHAINLOG_MAX); + if(paramTarget.targetLength > ZSTD_TARGETLENGTH_MAX) { + DISPLAY("INVALID PARAMETER CONSTRAINTS\n"); + return 0; + } if(paramTarget.strategy > ZSTD_btultra) { DISPLAY("INVALID PARAMETER CONSTRAINTS\n"); return 0; @@ -232,21 +240,68 @@ static int cParamValid(ZSTD_compressionParameters paramTarget) { return 1; } +//TODO: let targetLength = 0; static void cParamZeroMin(ZSTD_compressionParameters* paramTarget) { paramTarget->windowLog = paramTarget->windowLog ? paramTarget->windowLog : ZSTD_WINDOWLOG_MIN; paramTarget->searchLog = paramTarget->searchLog ? paramTarget->searchLog : ZSTD_SEARCHLOG_MIN; paramTarget->chainLog = paramTarget->chainLog ? paramTarget->chainLog : ZSTD_CHAINLOG_MIN; paramTarget->hashLog = paramTarget->hashLog ? paramTarget->hashLog : ZSTD_HASHLOG_MIN; paramTarget->searchLength = paramTarget->searchLength ? paramTarget->searchLength : ZSTD_SEARCHLENGTH_MIN; - paramTarget->targetLength = paramTarget->targetLength ? paramTarget->targetLength : 1; + paramTarget->targetLength = paramTarget->targetLength ? paramTarget->targetLength : 0; } -static void BMK_translateAdvancedParams(ZSTD_compressionParameters params) +static void BMK_translateAdvancedParams(const ZSTD_compressionParameters params) { DISPLAY("--zstd=windowLog=%u,chainLog=%u,hashLog=%u,searchLog=%u,searchLength=%u,targetLength=%u,strategy=%u \n", params.windowLog, params.chainLog, params.hashLog, params.searchLog, params.searchLength, params.targetLength, (U32)(params.strategy)); } +/* checks results are feasible */ +static int feasible(const BMK_result_t results, const constraint_t target) { + return (results.cSpeed >= target.cSpeed) && (results.dSpeed >= target.dSpeed) && (results.cMem <= target.cMem || !target.cMem); +} + +#define EPSILON 0.01 +static int epsilonEqual(const double c1, const double c2) { + return MAX(c1/c2,c2/c1) < 1 + EPSILON; +} + +/* checks exact equivalence to 0, to stop compiler complaining fpeq */ +static int eqZero(const double c1) { + return (U64)c1 == (U64)0.0 || (U64)c1 == (U64)-0.0; +} + +/* returns 1 if result2 is strictly 'better' than result1 */ +/* strict comparison / cutoff based */ +static int objective_lt(const BMK_result_t result1, const BMK_result_t result2) { + return (result1.cSize > result2.cSize) || (epsilonEqual(result1.cSize, result2.cSize) && result2.cSpeed > result1.cSpeed) + || (epsilonEqual(result1.cSize,result2.cSize) && epsilonEqual(result2.cSpeed, result1.cSpeed) && result2.dSpeed > result1.dSpeed); +} + +/* hill climbing value for part 1 */ +static double resultScore(const BMK_result_t res, const size_t srcSize, const constraint_t target) { + double cs = 0., ds = 0., rt, cm = 0.; + const double r1 = 1, r2 = 0.1, rtr = 0.5; + double ret; + if(target.cSpeed) { cs = res.cSpeed / (double)target.cSpeed; } + if(target.dSpeed) { ds = res.dSpeed / (double)target.dSpeed; } + if(target.cMem != (U32)-1) { cm = (double)target.cMem / res.cMem; } + rt = ((double)srcSize / res.cSize); + + ret = (MIN(1, cs) + MIN(1, ds) + MIN(1, cm))*r1 + rt * rtr + + (MAX(0, log(cs))+ MAX(0, log(ds))+ MAX(0, log(cm))) * r2; + //DISPLAY("resultScore: %f\n", ret); + return ret; +} + +/* factor sort of arbitrary */ +static constraint_t relaxTarget(constraint_t target) { + target.cMem = (U32)-1; + target.cSpeed *= 0.9; + target.dSpeed *= 0.9; + return target; +} + /*-******************************************************* * Bench functions *********************************************************/ @@ -268,9 +323,21 @@ const char* g_stratName[ZSTD_btultra+1] = { "ZSTD_greedy ", "ZSTD_lazy ", "ZSTD_lazy2 ", "ZSTD_btlazy2 ", "ZSTD_btopt ", "ZSTD_btultra "}; -/* TODO: support additional parameters (more files, fileSizes) */ static size_t BMK_benchParam(BMK_result_t* resultPtr, + const void* srcBuffer, const size_t srcSize, + const size_t* fileSizes, const unsigned nbFiles, + ZSTD_CCtx* ctx, ZSTD_DCtx* dctx, + const ZSTD_compressionParameters cParams) { + + BMK_return_t res = BMK_benchMem(srcBuffer,srcSize, fileSizes, nbFiles, 0, &cParams, NULL, 0, ctx, dctx, 0, "File"); + *resultPtr = res.result; + return res.error; +} + +/* benchParam but only takes in one file. */ +static size_t +BMK_benchParam1(BMK_result_t* resultPtr, const void* srcBuffer, size_t srcSize, ZSTD_CCtx* ctx, ZSTD_DCtx* dctx, const ZSTD_compressionParameters cParams) { @@ -280,27 +347,54 @@ BMK_benchParam(BMK_result_t* resultPtr, return res.error; } -static void BMK_printWinner(FILE* f, U32 cLevel, BMK_result_t result, ZSTD_compressionParameters params, size_t srcSize) -{ - char lvlstr[15] = "Custom Level"; - DISPLAY("\r%79s\r", ""); - fprintf(f," {%3u,%3u,%3u,%3u,%3u,%3u, %s }, ", - params.windowLog, params.chainLog, params.hashLog, params.searchLog, params.searchLength, - params.targetLength, g_stratName[(U32)(params.strategy)]); - if(cLevel != CUSTOM_LEVEL) { - snprintf(lvlstr, 15, " Level %2u ", cLevel); - } - fprintf(f, - "/* %s */ /* R:%5.3f at %5.1f MB/s - %5.1f MB/s */\n", - lvlstr, (double)srcSize / result.cSize, result.cSpeed / 1000000., result.dSpeed / 1000000.); -} - - typedef struct { BMK_result_t result; ZSTD_compressionParameters params; } winnerInfo_t; +/* global winner used for display. */ +//Should be totally 0 initialized? +static winnerInfo_t g_winner; //TODO: ratio is infinite at initialization, instead of 0 +static constraint_t g_targetConstraints; + +static void BMK_printWinner(FILE* f, const U32 cLevel, const BMK_result_t result, const ZSTD_compressionParameters params, const size_t srcSize) +{ + if(DEBUG || (objective_lt(g_winner.result, result) && feasible(result, g_targetConstraints))) { + char lvlstr[15] = "Custom Level"; + const U64 time = UTIL_clockSpanNano(g_time); + const U64 minutes = time / (60ULL * TIMELOOP_NANOSEC); + if(DEBUG && (objective_lt(g_winner.result, result) && feasible(result, g_targetConstraints))) { + DISPLAY("New Winner: \n"); + } + + DISPLAY("\r%79s\r", ""); + + fprintf(f," {%3u,%3u,%3u,%3u,%3u,%3u, %s }, ", + params.windowLog, params.chainLog, params.hashLog, params.searchLog, params.searchLength, + params.targetLength, g_stratName[(U32)(params.strategy)]); + if(cLevel != CUSTOM_LEVEL) { + snprintf(lvlstr, 15, " Level %2u ", cLevel); + } + fprintf(f, + "/* %s */ /* R:%5.3f at %5.1f MB/s - %5.1f MB/s */", + lvlstr, (double)srcSize / result.cSize, result.cSpeed / 1000000., result.dSpeed / 1000000.); + + if(TIMED) { fprintf(f, " - %lu:%lu:%05.2f", (unsigned long) minutes / 60,(unsigned long) minutes % 60, (double)(time - minutes * TIMELOOP_NANOSEC * 60ULL)/TIMELOOP_NANOSEC); } + fprintf(f, "\n"); + if(objective_lt(g_winner.result, result) && feasible(result, g_targetConstraints)) { + BMK_translateAdvancedParams(params); + g_winner.result = result; + g_winner.params = params; + } + } + //else { + // DISPLAY("G_WINNER: "); + // DISPLAY("/* R:%5.3f at %5.1f MB/s - %5.1f MB/s */ \n",(double)srcSize / g_winner.result.cSize , g_winner.result.cSpeed / 1000000 , g_winner.result.dSpeed / 1000000); + // DISPLAY("LOSER : "); + // DISPLAY("/* R:%5.3f at %5.1f MB/s - %5.1f MB/s */ \n",(double)srcSize / result.cSize, result.cSpeed / 1000000 , result.dSpeed / 1000000); + //} +} + static void BMK_printWinners2(FILE* f, const winnerInfo_t* winners, size_t srcSize) { int cLevel; @@ -358,7 +452,7 @@ static int BMK_seed(winnerInfo_t* winners, const ZSTD_compressionParameters para int better = 0; int cLevel; - BMK_benchParam(&testResult, srcBuffer, srcSize, ctx, dctx, params); + BMK_benchParam1(&testResult, srcBuffer, srcSize, ctx, dctx, params); for (cLevel = 1; cLevel <= NB_LEVELS_TRACKED; cLevel++) { @@ -470,7 +564,7 @@ static ZSTD_compressionParameters sanitizeParams(ZSTD_compressionParameters para /* new length */ /* keep old array, will need if iter over strategy. */ -static int sanitizeVarArray(int varLength, U32* varArray, U32* varNew, ZSTD_strategy strat) { +static int sanitizeVarArray(const int varLength, const U32* varArray, U32* varNew, const ZSTD_strategy strat) { int i, j = 0; for(i = 0; i < varLength; i++) { if( !((varArray[i] == CLOG_IND && strat == ZSTD_fast) @@ -515,53 +609,10 @@ static int variableParams(const ZSTD_compressionParameters paramConstraints, U32 return j; } -/* computes inverse of above array, returns same number, -1 = unused ind */ -static int inverseVariableParams(const ZSTD_compressionParameters paramConstraints, U32* res) { - int j = 0; - if(!paramConstraints.windowLog) { - res[WLOG_IND] = j; - j++; - } else { - res[WLOG_IND] = -1; - } - if(!paramConstraints.chainLog) { - res[j] = CLOG_IND; - j++; - } else { - res[CLOG_IND] = -1; - } - if(!paramConstraints.hashLog) { - res[j] = HLOG_IND; - j++; - } else { - res[HLOG_IND] = -1; - } - if(!paramConstraints.searchLog) { - res[j] = SLOG_IND; - j++; - } else { - res[SLOG_IND] = -1; - } - if(!paramConstraints.searchLength) { - res[j] = SLEN_IND; - j++; - } else { - res[SLEN_IND] = -1; - } - if(!paramConstraints.targetLength) { - res[j] = TLEN_IND; - j++; - } else { - res[TLEN_IND] = -1; - } - - return j; -} - /* amt will probably always be \pm 1? */ /* slight change from old paramVariation, targetLength can only take on powers of 2 now (999 ~= 1024?) */ /* take max/min bounds into account as well? */ -static void paramVaryOnce(U32 paramIndex, int amt, ZSTD_compressionParameters* ptr) { +static void paramVaryOnce(const U32 paramIndex, const int amt, ZSTD_compressionParameters* ptr) { switch(paramIndex) { case WLOG_IND: ptr->windowLog += amt; break; @@ -571,8 +622,14 @@ static void paramVaryOnce(U32 paramIndex, int amt, ZSTD_compressionParameters* p case SLEN_IND: ptr->searchLength += amt; break; case TLEN_IND: if(amt >= 0) { - ptr->targetLength <<= amt; - ptr->targetLength = MIN(ptr->targetLength, 999); + if(ptr->targetLength == 0) { + if(amt > 0) { + ptr->targetLength = MIN(1 << (amt - 1), 999); + } + } else { + ptr->targetLength <<= amt; + ptr->targetLength = MIN(ptr->targetLength, 999); + } } else { if(ptr->targetLength == 999) { ptr->targetLength = 1024; @@ -584,11 +641,8 @@ static void paramVaryOnce(U32 paramIndex, int amt, ZSTD_compressionParameters* p } } -//Don't fuzz fixed variables. -//turn pcs to pcs array with macro for params. -//pass in variation array from variableParams -//take nbChanges as argument? -static void paramVariation(ZSTD_compressionParameters* ptr, const U32* varyParams, const int varyLen, U32 nbChanges) +/* varies ptr by nbChanges respecting varyParams*/ +static void paramVariation(ZSTD_compressionParameters* ptr, const U32* varyParams, const int varyLen, const U32 nbChanges) { ZSTD_compressionParameters p; U32 validated = 0; @@ -600,19 +654,11 @@ static void paramVariation(ZSTD_compressionParameters* ptr, const U32* varyParam paramVaryOnce(varyParams[changeID >> 1], ((changeID & 1) << 1) - 1, &p); } validated = !ZSTD_isError(ZSTD_checkCParams(p)); - //validated = cParamValid(p); - - //Make sure memory is at least close to feasible? - //ZSTD_estimateCCtxSize thing. } - *ptr = sanitizeParams(p); + *ptr = p;//sanitizeParams(p); } -//varyParams gives us table size? -//1 per strategy -//varyParams should always be sorted smallest to largest -//take arrayLen to allocate memotable -//should be ~10^7 unconstrained. +/* length of memo table given free variables */ static size_t memoTableLen(const U32* varyParams, const int varyLen) { size_t arrayLen = 1; int i; @@ -622,11 +668,14 @@ static size_t memoTableLen(const U32* varyParams, const int varyLen) { return arrayLen; } -//sort of ~lg2 (replace 1024 w/ 999) for memoTableInd Tlen +//sort of ~lg2 (replace 1024 w/ 999, and add 0 at lower end of range) for memoTableInd Tlen static unsigned lg2(unsigned x) { - unsigned j = 0; + unsigned j = 1; if(x == 999) { - return 10; + return 11; + } + if(!x) { + return 0; } while(x >>= 1) { j++; @@ -634,8 +683,7 @@ static unsigned lg2(unsigned x) { return j; } -//indexes compressionParameters into memotable -//of form +/* returns unique index of compression parameters */ static unsigned memoTableInd(const ZSTD_compressionParameters* ptr, const U32* varyParams, const int varyLen) { int i; unsigned ind = 0; @@ -652,25 +700,24 @@ static unsigned memoTableInd(const ZSTD_compressionParameters* ptr, const U32* v return ind; } -/* presumably, the unfilled parameters are already at their correct value */ -/* inverse above function for varyParams */ +/* inverse of above function (from index to parameters) */ static void memoTableIndInv(ZSTD_compressionParameters* ptr, const U32* varyParams, const int varyLen, size_t ind) { int i; for(i = varyLen - 1; i >= 0; i--) { switch(varyParams[i]) { - case WLOG_IND: ptr->windowLog = ind % WLOG_RANGE + ZSTD_WINDOWLOG_MIN; ind /= WLOG_RANGE; break; - case CLOG_IND: ptr->chainLog = ind % CLOG_RANGE + ZSTD_CHAINLOG_MIN; ind /= CLOG_RANGE; break; - case HLOG_IND: ptr->hashLog = ind % HLOG_RANGE + ZSTD_HASHLOG_MIN; ind /= HLOG_RANGE; break; - case SLOG_IND: ptr->searchLog = ind % SLOG_RANGE + ZSTD_SEARCHLOG_MIN; ind /= SLOG_RANGE; break; - case SLEN_IND: ptr->searchLength = ind % SLEN_RANGE + ZSTD_SEARCHLENGTH_MIN; ind /= SLEN_RANGE; break; - case TLEN_IND: ptr->targetLength = MIN(1 << (ind % TLEN_RANGE), 999); ind /= TLEN_RANGE; break; + case WLOG_IND: ptr->windowLog = ind % WLOG_RANGE + ZSTD_WINDOWLOG_MIN; ind /= WLOG_RANGE; break; + case CLOG_IND: ptr->chainLog = ind % CLOG_RANGE + ZSTD_CHAINLOG_MIN; ind /= CLOG_RANGE; break; + case HLOG_IND: ptr->hashLog = ind % HLOG_RANGE + ZSTD_HASHLOG_MIN; ind /= HLOG_RANGE; break; + case SLOG_IND: ptr->searchLog = ind % SLOG_RANGE + ZSTD_SEARCHLOG_MIN; ind /= SLOG_RANGE; break; + case SLEN_IND: ptr->searchLength = ind % SLEN_RANGE + ZSTD_SEARCHLENGTH_MIN; ind /= SLEN_RANGE; break; + case TLEN_IND: ptr->targetLength = (ind % TLEN_RANGE) ? MIN(1 << ((ind % TLEN_RANGE) - 1), 999) : 0; ind /= TLEN_RANGE; break; } } } -//initializing memoTable -/* */ -static void memoTableInit(U8* memoTable, ZSTD_compressionParameters paramConstraints, constraint_t target, const U32* varyParams, const int varyLen, const size_t srcSize) { + +/* Initialize memotable, immediately mark redundant / obviously infeasible params as */ +static void memoTableInit(U8* memoTable, ZSTD_compressionParameters paramConstraints, const constraint_t target, const U32* varyParams, const int varyLen, const size_t srcSize) { size_t i; size_t arrayLen = memoTableLen(varyParams, varyLen); int cwFixed = !paramConstraints.chainLog || !paramConstraints.windowLog; @@ -682,13 +729,11 @@ static void memoTableInit(U8* memoTable, ZSTD_compressionParameters paramConstra for(i = 0; i < arrayLen; i++) { memoTableIndInv(¶mConstraints, varyParams, varyLen, i); - if(ZSTD_estimateCCtxSize_usingCParams(paramConstraints) + (1 << paramConstraints.windowLog) > target.cMem) { - //infeasible; + if((ZSTD_estimateCCtxSize_usingCParams(paramConstraints) + (1ULL << paramConstraints.windowLog)) > (size_t)target.cMem + (size_t)(target.cMem / 10)) { memoTable[i] = 255; j++; } - //TODO: remove any where memoTable wlog is mark any where windowlog is too big for data. - if(wFixed && (1 << paramConstraints.windowLog) > (srcSize << 1)) { + if(wFixed && (1ULL << paramConstraints.windowLog) > (srcSize << 1)) { memoTable[i] = 255; } /* nil out parameter sets equivalent to others. */ @@ -712,7 +757,42 @@ static void memoTableInit(U8* memoTable, ZSTD_compressionParameters paramConstra } } } - DISPLAY("%d / %d Invalid\n", j, (int)i); + DEBUGOUTPUT("%d / %d Invalid\n", j, (int)i); + if((int)i == j) { + DEBUGOUTPUT("!!!Strategy %d totally infeasible\n", (int)paramConstraints.strategy) + } +} + +/* inits memotables for all (including mallocs), all strategies */ +/* takes unsanitized varyParams */ + +//TODO: check for errors/nulls +static U8** memoTableInitAll(ZSTD_compressionParameters paramConstraints, constraint_t target, const U32* varyParams, const int varyLen, const size_t srcSize) { + U32 varNew[NUM_PARAMS]; + int varLenNew; + U8** mtAll = malloc(sizeof(U8*) * (ZSTD_btultra + 1)); + int i; + if(mtAll == NULL) { + return NULL; + } + for(i = 1; i <= (int)ZSTD_btultra; i++) { + varLenNew = sanitizeVarArray(varyLen, varyParams, varNew, i); + mtAll[i] = malloc(sizeof(U8) * memoTableLen(varNew, varLenNew)); + if(mtAll[i] == NULL) { + return NULL; + } + memoTableInit(mtAll[i], paramConstraints, target, varNew, varLenNew, srcSize); + } + return mtAll; +} + +static void memoTableFreeAll(U8** mtAll) { + int i; + if(mtAll == NULL) { return; } + for(i = 1; i <= (int)ZSTD_btultra; i++) { + free(mtAll[i]); + } + free(mtAll); } #define PARAMTABLELOG 25 @@ -782,10 +862,7 @@ static ZSTD_compressionParameters randomParams(void) return p; } -//destructively modifies pc. -//Maybe if memoTable[ind] > 0 too often, count zeroes and explicitly choose from free stuff? -//^ maybe this doesn't matter, with |mt| size it has \approx 1-(1/e) of finding even single free spot in |mt| tries, not too bad. -//TODO: maybe memoTable pc before sanitization too so no repeats? +/* Sets pc to random unmeasured set of parameters */ static void randomConstrainedParams(ZSTD_compressionParameters* pc, U32* varArray, int varLen, U8* memoTable) { int tries = memoTableLen(varArray, varLen); //configurable, @@ -795,9 +872,7 @@ static void randomConstrainedParams(ZSTD_compressionParameters* pc, U32* varArra ind = (FUZ_rand(&g_rand)) % maxSize; tries--; } while(memoTable[ind] > 0 && tries > 0); - //&& FUZ_rand(&g_rand) % 256 > memoTable[ind]); get nd choosing? (helpful w/ distance) /* maybe > infeasible bound? */ - /* memoTable[ind] == 0 -> unexplored */ memoTableIndInv(pc, varArray, varLen, (unsigned)ind); *pc = sanitizeParams(*pc); } @@ -822,7 +897,7 @@ static void BMK_benchOnce(ZSTD_CCtx* cctx, ZSTD_DCtx* dctx, const void* srcBuffe { BMK_result_t testResult; g_params = ZSTD_adjustCParams(g_params, srcSize, 0); - BMK_benchParam(&testResult, srcBuffer, srcSize, cctx, dctx, g_params); + BMK_benchParam1(&testResult, srcBuffer, srcSize, cctx, dctx, g_params); DISPLAY("Compression Ratio: %.3f Compress Speed: %.1f MB/s Decompress Speed: %.1f MB/s\n", (double)srcSize / testResult.cSize, testResult.cSpeed / 1000000, testResult.dSpeed / 1000000); return; @@ -847,7 +922,7 @@ static void BMK_benchFullTable(ZSTD_CCtx* cctx, ZSTD_DCtx* dctx, const void* src /* baseline config for level 1 */ ZSTD_compressionParameters const l1params = ZSTD_getCParams(1, blockSize, 0); BMK_result_t testResult; - BMK_benchParam(&testResult, srcBuffer, srcSize, cctx, dctx, l1params); + BMK_benchParam1(&testResult, srcBuffer, srcSize, cctx, dctx, l1params); BMK_init_level_constraints((int)((testResult.cSpeed * 31) / 32)); } @@ -974,112 +1049,14 @@ int benchFiles(const char** fileNamesTable, int nbFiles) return 0; } -//parameter feasibility is not checked, should just be restricted from use. -static int feasible(BMK_result_t results, constraint_t target) { - return (results.cSpeed >= target.cSpeed) && (results.dSpeed >= target.dSpeed) && (results.cMem <= target.cMem || !target.cMem); -} -#define EPSILON 0.01 -static int epsilonEqual(double c1, double c2) { - return MAX(c1/c2,c2/c1) < 1 + EPSILON; -} - -//so the compiler stops warning -static int eqZero(double c1) { - return (U64)c1 == (U64)0.0 || (U64)c1 == (U64)-0.0; -} - -/* returns 1 if result2 is strictly 'better' than result1 */ -/* strict comparison / cutoff based */ -static int objective_lt(BMK_result_t result1, BMK_result_t result2) { - return (result1.cSize > result2.cSize) || (epsilonEqual(result1.cSize, result2.cSize) && result2.cSpeed > result1.cSpeed) - || (epsilonEqual(result1.cSize,result2.cSize) && epsilonEqual(result2.cSpeed, result1.cSpeed) && result2.dSpeed > result1.dSpeed); -} - -//will probably be some linear combinartion of comp speed, decompSpeed, & ratio (maybe size), and memory? -//pretty arbitrary right now -//maybe better - higher coefficient when below threshold, lower when above -//need to normalize speed? or just use ratio speed / target? -//Maybe don't use ratio at all when looking for feasibility? - -/* maybe dynamically vary the coefficients for this around based on what's already been discovered. (maybe make a reversed ratio cutoff?) concave to pheaily penalize below ratio? */ -static double resultScore(BMK_result_t res, size_t srcSize, constraint_t target) { - double cs = 0., ds = 0., rt, cm = 0.; - const double r1 = 1, r2 = 0.1, rtr = 0.5; - double ret; - if(target.cSpeed) { cs = res.cSpeed / (double)target.cSpeed; } - if(target.dSpeed) { ds = res.dSpeed / (double)target.dSpeed; } - if(target.cMem != (U32)-1) { cm = (double)target.cMem / res.cMem; } - rt = ((double)srcSize / res.cSize); - - //(void)rt; - //(void)rtr; - - ret = (MIN(1, cs) + MIN(1, ds) + MIN(1, cm))*r1 + rt * rtr + - (MAX(0, log(cs))+ MAX(0, log(ds))+ MAX(0, log(cm))) * r2; - //DISPLAY("resultScore: %f\n", ret); - return ret; -} - -/* - double W_ratio = (double)srcSize / testResult.cSize; - double O_ratio = (double)srcSize / winners[cLevel].result.cSize; - double W_ratioNote = log (W_ratio); - double O_ratioNote = log (O_ratio); - size_t W_DMemUsed = (1 << params.windowLog) + (16 KB); - size_t O_DMemUsed = (1 << winners[cLevel].params.windowLog) + (16 KB); - double W_DMemUsed_note = W_ratioNote * ( 40 + 9*cLevel) - log((double)W_DMemUsed); - double O_DMemUsed_note = O_ratioNote * ( 40 + 9*cLevel) - log((double)O_DMemUsed); - - size_t W_CMemUsed = (1 << params.windowLog) + ZSTD_estimateCCtxSize_usingCParams(params); - size_t O_CMemUsed = (1 << winners[cLevel].params.windowLog) + ZSTD_estimateCCtxSize_usingCParams(winners[cLevel].params); - double W_CMemUsed_note = W_ratioNote * ( 50 + 13*cLevel) - log((double)W_CMemUsed); - double O_CMemUsed_note = O_ratioNote * ( 50 + 13*cLevel) - log((double)O_CMemUsed); - - double W_CSpeed_note = W_ratioNote * ( 30 + 10*cLevel) + log(testResult.cSpeed); - double O_CSpeed_note = O_ratioNote * ( 30 + 10*cLevel) + log(winners[cLevel].result.cSpeed); - - double W_DSpeed_note = W_ratioNote * ( 20 + 2*cLevel) + log(testResult.dSpeed); - double O_DSpeed_note = O_ratioNote * ( 20 + 2*cLevel) + log(winners[cLevel].result.dSpeed); - -*/ -//ratio tradeoffs, may be useful in guiding - -/* objective_lt, but based on scoring function */ -static int objective_lt2(BMK_result_t result1, BMK_result_t result2, size_t srcSize, constraint_t target) { - return resultScore(result1, srcSize, target) < resultScore(result2, srcSize, target); -} - -/* res gives array dimensions, should be size NUM_PARAMS */ -static size_t computeStateSize(const ZSTD_compressionParameters paramConstraints, U32* res) { - int ind = 0; - size_t base = 1; - if(!paramConstraints.windowLog) { res[ind] = ZSTD_WINDOWLOG_MAX - ZSTD_WINDOWLOG_MIN + 1; base *= res[ind]; ind++; } - if(!paramConstraints.chainLog) { res[ind] = ZSTD_CHAINLOG_MAX - ZSTD_CHAINLOG_MIN + 1; base *= res[ind]; ind++; } - if(!paramConstraints.hashLog) { res[ind] = ZSTD_HASHLOG_MAX - ZSTD_HASHLOG_MIN + 1; base *= res[ind]; ind++; } - if(!paramConstraints.searchLog) { res[ind] = ZSTD_SEARCHLOG_MAX - ZSTD_SEARCHLOG_MIN + 1; base *= res[ind]; ind++; } - if(!paramConstraints.searchLength) { res[ind] = ZSTD_SEARCHLENGTH_MAX - ZSTD_SEARCHLENGTH_MIN + 1; base *= res[ind]; ind++; } - if(!paramConstraints.targetLength) { res[ind] = 11; base *= res[ind]; ind++; } //restricting from 2^[0,10], no such macros - if(!(U32)paramConstraints.strategy) { res[ind] = 8; base *= 8; } //not strictly true, maybe would want to case on this. - - return base; -} - - -static unsigned calcViolation(BMK_result_t results, constraint_t target) { - int diffcSpeed = MAX(target.cSpeed - results.cSpeed, 0); - int diffdSpeed = MAX(target.dSpeed - results.dSpeed, 0); - int diffcMem = MAX(results.cMem - target.cMem, 0); - return diffcSpeed + diffdSpeed + diffcMem; -} /* -uncertaintyConstant >= 1 -returns -1 = 'certainly' infeasible - 0 = unceratin - 1 = 'certainly' feasible +checks feasibility with uncertainty. +-1 : certainly infeasible + 0 : uncertain + 1 : certainly feasible */ -//paramTarget misnamed, should just be target static int uncertainFeasibility(double const uncertaintyConstantC, double const uncertaintyConstantD, const constraint_t paramTarget, const BMK_result_t* const results) { if((paramTarget.cSpeed != 0 && results->cSpeed * uncertaintyConstantC < paramTarget.cSpeed) || (paramTarget.dSpeed != 0 && results->dSpeed * uncertaintyConstantD < paramTarget.dSpeed) || @@ -1099,12 +1076,8 @@ static int uncertainFeasibility(double const uncertaintyConstantC, double const -1 - worse assume prev_best status is run fully? but then we'd have to rerun any winners anyway */ -//presumably memory has already been compared, mostly worried about mem, cspeed, dspeed -//uncertainty only applies to speed. -//if using objective fn, this could be much easier since we could just scale that. -//difficult to make judgements about later parameters in prioritization type when there's -//uncertainty on the first. -static int uncertainComparison(double const uncertaintyConstantC, double const uncertaintyConstantD, BMK_result_t* candidate, BMK_result_t* prevBest) { +/* not as useful as initially believed */ +static int uncertainComparison(double const uncertaintyConstantC, double const uncertaintyConstantD, const BMK_result_t* candidate, const BMK_result_t* prevBest) { (void)uncertaintyConstantD; //unused for now if(candidate->cSpeed > prevBest->cSpeed * uncertaintyConstantC) { return 1; @@ -1115,34 +1088,21 @@ static int uncertainComparison(double const uncertaintyConstantC, double const u } } -/* speed in b, srcSize in b/s loopDuration in ns */ -//TODO: simplify code in feasibleBench with this instead of writing it all out. -//only applicable for single loop -static double calcUncertainty(double speed, size_t srcSize) { - U64 loopDuration; - if(eqZero(speed)) { return 2; } - loopDuration = ((srcSize * TIMELOOP_NANOSEC) / speed); - return MIN((loopDuration + (double)2 * g_clockGranularity) / loopDuration, 2); -} +/*benchmarks and tests feasibility together + 1 = true = better + 0 = false = not better + if true then resultPtr will give results. + 2+ on error? */ -//benchmarks and tests feasibility together -//1 = true = better -//0 = false = not better -//if true then resultPtr will give results. -//2+ on error? -//alt: error = 0 / infeasible as well; -//maybe use compress_only mode for ratio-finding benchmark? -//prioritize ratio > cSpeed > dSpeed > cMem -//Misnamed - should be worse, better, error -//alternative (to make work for feasible-pt searching as well) - only compare to winner, not to target -//but then we need to judge what better means in this context, which shouldn't be the same (strict ratio improvement) +//Maybe use compress_only for benchmark #define INFEASIBLE_RESULT 0 #define FEASIBLE_RESULT 1 #define ERROR_RESULT 2 static int feasibleBench(BMK_result_t* resultPtr, - const void* srcBuffer, size_t srcSize, - void* dstBuffer, size_t dstSize, - void* dictBuffer, size_t dictSize, + const void* srcBuffer, const size_t srcSize, + void* dstBuffer, const size_t dstSize, + void* dictBuffer, const size_t dictSize, + const size_t* fileSizes, const size_t nbFiles, ZSTD_CCtx* ctx, ZSTD_DCtx* dctx, const ZSTD_compressionParameters cParams, const constraint_t target, @@ -1156,8 +1116,8 @@ static int feasibleBench(BMK_result_t* resultPtr, //alternative - test 1 iter for ratio, (possibility of error 3 which is fine), //maybe iter this until 2x measurable for better guarantee? - DISPLAY("Feas:\n"); - benchres = BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, &srcSize, 1, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); + DEBUGOUTPUT("Feas:\n"); + benchres = BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, fileSizes, nbFiles, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); if(benchres.error) { DISPLAY("ERROR %d !!\n", benchres.error); } @@ -1174,7 +1134,7 @@ static int feasibleBench(BMK_result_t* resultPtr, //problem - tested in fullbench, saw speed vary 3x between iters, maybe raise uncertaintyConstraint up? //possibly has to do with initCCtx? or system stuff? //asymmetric +/- constant needed? - uncertaintyConstantC = MIN((loopDurationC + (double)(2 * g_clockGranularity)/loopDurationC), 2); //.02 seconds + uncertaintyConstantC = MIN((loopDurationC + (double)(2 * g_clockGranularity)/loopDurationC) * 1.1, 3); //.02 seconds } if(eqZero(benchres.result.dSpeed)) { loopDurationD = 0; @@ -1184,20 +1144,41 @@ static int feasibleBench(BMK_result_t* resultPtr, //problem - tested in fullbench, saw speed vary 3x between iters, maybe raise uncertaintyConstraint up? //possibly has to do with initCCtx? or system stuff? //asymmetric +/- constant needed? - uncertaintyConstantD = MIN((loopDurationD + (double)(2 * g_clockGranularity)/loopDurationD), 2); //.02 seconds + uncertaintyConstantD = MIN((loopDurationD + (double)(2 * g_clockGranularity)/loopDurationD) * 1.1, 3); //.02 seconds } if(benchres.result.cSize < winnerResult->cSize) { //better compression ratio, just needs to be feasible - //optimistic assume speed - //incoporate some sort of tradeoff comparison with the winner's results? - int feas = uncertainFeasibility(uncertaintyConstantC, uncertaintyConstantD, target, &(benchres.result)); + int feas; + if(loopDurationC < TIMELOOP_NANOSEC / 10) { + BMK_return_t benchres2; + adv.mode = BMK_compressOnly; + benchres2 = BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, fileSizes, nbFiles, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); + if(benchres2.error) { + return ERROR_RESULT; + } else { + benchres = benchres2; + } + } + if(loopDurationD < TIMELOOP_NANOSEC / 10) { + BMK_return_t benchres2; + adv.mode = BMK_decodeOnly; + benchres2 = BMK_benchMemAdvanced(dstBuffer, dstSize, NULL, 0, &benchres.result.cSize, 1, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); + if(benchres2.error) { + return ERROR_RESULT; + } else { + benchres.result.dSpeed = benchres2.result.dSpeed; + } + } + *resultPtr = benchres.result; + + feas = uncertainFeasibility(uncertaintyConstantC, uncertaintyConstantD, target, &(benchres.result)); if(feas == 0) { // uncertain feasibility adv.loopMode = BMK_timeMode; if(loopDurationC < TIMELOOP_NANOSEC) { BMK_return_t benchres2; adv.mode = BMK_compressOnly; - benchres2 = BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, &srcSize, 1, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); + benchres2 = BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, fileSizes, nbFiles, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); if(benchres2.error) { return ERROR_RESULT; } else { @@ -1220,13 +1201,34 @@ static int feasibleBench(BMK_result_t* resultPtr, return (feas + 1) >> 1; //relies on INFEASIBLE_RESULT == 0, FEASIBLE_RESULT == 1 } } else if (benchres.result.cSize == winnerResult->cSize) { //equal ratio, needs to be better than winner in cSpeed/ dSpeed / cMem - int feas = uncertainFeasibility(uncertaintyConstantC, uncertaintyConstantD, target, &(benchres.result)); + int feas; + if(loopDurationC < TIMELOOP_NANOSEC / 10) { + BMK_return_t benchres2; + adv.mode = BMK_compressOnly; + benchres2 = BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, fileSizes, nbFiles, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); + if(benchres2.error) { + return ERROR_RESULT; + } else { + benchres = benchres2; + } + } + if(loopDurationD < TIMELOOP_NANOSEC / 10) { + BMK_return_t benchres2; + adv.mode = BMK_decodeOnly; + benchres2 = BMK_benchMemAdvanced(dstBuffer, dstSize, NULL, 0, &benchres.result.cSize, 1, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); + if(benchres2.error) { + return ERROR_RESULT; + } else { + benchres.result.dSpeed = benchres2.result.dSpeed; + } + } + feas = uncertainFeasibility(uncertaintyConstantC, uncertaintyConstantD, target, &(benchres.result)); if(feas == 0) { // uncertain feasibility adv.loopMode = BMK_timeMode; if(loopDurationC < TIMELOOP_NANOSEC) { BMK_return_t benchres2; adv.mode = BMK_compressOnly; - benchres2 = BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, &srcSize, 1, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); + benchres2 = BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, fileSizes, nbFiles, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); if(benchres2.error) { return ERROR_RESULT; } else { @@ -1252,7 +1254,7 @@ static int feasibleBench(BMK_result_t* resultPtr, return INFEASIBLE_RESULT; } else { //possibly better, benchmark and find out adv.loopMode = BMK_timeMode; - benchres = BMK_benchMemAdvanced(srcBuffer, srcSize, dstBuffer, dstSize, &srcSize, 1, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); + benchres = BMK_benchMemAdvanced(srcBuffer, srcSize, dstBuffer, dstSize, fileSizes, nbFiles, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); *resultPtr = benchres.result; return objective_lt(*winnerResult, benchres.result); } @@ -1265,16 +1267,17 @@ static int feasibleBench(BMK_result_t* resultPtr, } else { return ERROR_RESULT; //BMK error } - } -//sameas before, but +/-? + +//same as before, but +/-? //alternative, just return comparison result, leave caller to worry about feasibility. //have version of benchMemAdvanced which takes in dstBuffer/cap as well? //(motivation: repeat tests (maybe just on decompress) don't need further compress runs) static int infeasibleBench(BMK_result_t* resultPtr, - const void* srcBuffer, size_t srcSize, - void* dstBuffer, size_t dstSize, - void* dictBuffer, size_t dictSize, + const void* srcBuffer, const size_t srcSize, + void* dstBuffer, const size_t dstSize, + void* dictBuffer, const size_t dictSize, + const size_t* fileSizes, const size_t nbFiles, ZSTD_CCtx* ctx, ZSTD_DCtx* dctx, const ZSTD_compressionParameters cParams, const constraint_t target, @@ -1288,19 +1291,10 @@ static int infeasibleBench(BMK_result_t* resultPtr, adv.loopMode = BMK_iterMode; //can only use this for ratio measurement then, super inaccurate timing adv.nbSeconds = 1; //get ratio and 2x approx speed? //maybe run until twice MIN(minloopinterval * clockDuration) - DISPLAY("WinnerScore: %f\n ", winnerRS); - /* - adv.loopMode = BMK_timeMode; - adv.nbSeconds = 1; */ - benchres = BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, &srcSize, 1, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); - BMK_printWinner(stdout, CUSTOM_LEVEL, benchres.result, cParams, srcSize); - benchres = BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, &srcSize, 1, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); - BMK_printWinner(stdout, CUSTOM_LEVEL, benchres.result, cParams, srcSize); + DEBUGOUTPUT("WinnerScore: %f\n ", winnerRS); - adv.loopMode = BMK_timeMode; - adv.nbSeconds = 1; - benchres = BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, &srcSize, 1, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); - BMK_printWinner(stdout, CUSTOM_LEVEL, benchres.result, cParams, srcSize); + benchres = BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, fileSizes, nbFiles, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); + BMK_printWinner(stdout, CUSTOM_LEVEL, benchres.result, cParams, srcSize); if(!benchres.error) { *resultPtr = benchres.result; @@ -1309,9 +1303,7 @@ static int infeasibleBench(BMK_result_t* resultPtr, uncertaintyConstantC = 2; } else { loopDurationC = ((srcSize * TIMELOOP_NANOSEC) / benchres.result.cSpeed); - //problem - tested in fullbench, saw speed vary 3x between iters, maybe raise uncertaintyConstraint up? - //possibly has to do with initCCtx? or system stuff? - uncertaintyConstantC = MIN((loopDurationC + (double)(2 * g_clockGranularity)/loopDurationC), 2); //.02 seconds + uncertaintyConstantC = MIN((loopDurationC + (double)(2 * g_clockGranularity)/loopDurationC * 1.1), 3); //.02 seconds } if(eqZero(benchres.result.dSpeed)) { @@ -1319,11 +1311,32 @@ static int infeasibleBench(BMK_result_t* resultPtr, uncertaintyConstantD = 2; } else { loopDurationD = ((srcSize * TIMELOOP_NANOSEC) / benchres.result.dSpeed); - //problem - tested in fullbench, saw speed vary 3x between iters, maybe raise uncertaintyConstraint up? - //possibly has to do with initCCtx? or system stuff? - uncertaintyConstantD = MIN((loopDurationD + (double)(2 * g_clockGranularity)/loopDurationD), 2); //.02 seconds + uncertaintyConstantD = MIN((loopDurationD + (double)(2 * g_clockGranularity)/loopDurationD) * 1.1 , 3); //.02 seconds } + + if(loopDurationC < TIMELOOP_NANOSEC / 10) { + BMK_return_t benchres2; + adv.mode = BMK_compressOnly; + benchres2 = BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, fileSizes, nbFiles, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); + if(benchres2.error) { + return ERROR_RESULT; + } else { + benchres = benchres2; + } + } + if(loopDurationD < TIMELOOP_NANOSEC / 10) { + BMK_return_t benchres2; + adv.mode = BMK_decodeOnly; + benchres2 = BMK_benchMemAdvanced(dstBuffer, dstSize, NULL, 0, &benchres.result.cSize, 1, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); + if(benchres2.error) { + return ERROR_RESULT; + } else { + benchres.result.dSpeed = benchres2.result.dSpeed; + } + } + *resultPtr = benchres.result; + /* benchres's certainty range. */ resultMax = benchres.result; resultMin = benchres.result; @@ -1331,8 +1344,6 @@ static int infeasibleBench(BMK_result_t* resultPtr, resultMax.dSpeed *= uncertaintyConstantD; resultMin.cSpeed /= uncertaintyConstantC; resultMin.dSpeed /= uncertaintyConstantD; - (void)resultMin; - //TODO: consider if resultMin is actually needed. if (winnerRS > resultScore(resultMax, srcSize, target)) { return INFEASIBLE_RESULT; } else { @@ -1341,7 +1352,7 @@ static int infeasibleBench(BMK_result_t* resultPtr, if(loopDurationC < TIMELOOP_NANOSEC) { BMK_return_t benchres2; adv.mode = BMK_compressOnly; - benchres2 = BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, &srcSize, 1, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); + benchres2 = BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, fileSizes, nbFiles, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); if(benchres2.error) { return ERROR_RESULT; } else { @@ -1350,9 +1361,7 @@ static int infeasibleBench(BMK_result_t* resultPtr, } if(loopDurationD < TIMELOOP_NANOSEC) { BMK_return_t benchres2; - adv.mode = BMK_decodeOnly; - //TODO: dstBuffer corrupted sometime between top and now - //probably occuring in feasible bench too. + adv.mode = BMK_decodeOnly; benchres2 = BMK_benchMemAdvanced(dstBuffer, dstSize, NULL, 0, &benchres.result.cSize, 1, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); if(benchres2.error) { return ERROR_RESULT; @@ -1372,28 +1381,27 @@ static int infeasibleBench(BMK_result_t* resultPtr, } /* wrap feasibleBench w/ memotable */ -//TODO: void sanitized and unsanitized ver's so input doesn't double-choose #define INFEASIBLE_THRESHOLD 200 static int feasibleBenchMemo(BMK_result_t* resultPtr, - const void* srcBuffer, size_t srcSize, - void* dstBuffer, size_t dstSize, - void* dictBuffer, size_t dictSize, + const void* srcBuffer, const size_t srcSize, + void* dstBuffer, const size_t dstSize, + void* dictBuffer, const size_t dictSize, + const size_t* fileSizes, const size_t nbFiles, ZSTD_CCtx* ctx, ZSTD_DCtx* dctx, const ZSTD_compressionParameters cParams, const constraint_t target, BMK_result_t* winnerResult, U8* memoTable, - U32* varyParams, const int varyLen) { + const U32* varyParams, const int varyLen) { - size_t memind = memoTableInd(&cParams, varyParams, varyLen); + const size_t memind = memoTableInd(&cParams, varyParams, varyLen); - //BMK_translateAdvancedParams(cParams); if(memoTable[memind] >= INFEASIBLE_THRESHOLD) { return INFEASIBLE_RESULT; //probably pick a different code for already tested? //maybe remove this if we incorporate nonrandom location picking? //what is the intended behavior in this case? //ignore? stop iterating completely? other? } else { - int res = feasibleBench(resultPtr, srcBuffer, srcSize, dstBuffer, dstSize, dictBuffer, dictSize, ctx, dctx, + int res = feasibleBench(resultPtr, srcBuffer, srcSize, dstBuffer, dstSize, dictBuffer, dictSize, fileSizes, nbFiles, ctx, dctx, cParams, target, winnerResult); memoTable[memind] = 255; //tested are all infeasible (other possible values for opti) return res; @@ -1403,21 +1411,21 @@ static int feasibleBenchMemo(BMK_result_t* resultPtr, //should infeasible stage searching also be memo-marked in the same way? //don't actually memoize unless result is feasible/error? static int infeasibleBenchMemo(BMK_result_t* resultPtr, - const void* srcBuffer, size_t srcSize, - void* dstBuffer, size_t dstSize, - void* dictBuffer, size_t dictSize, + const void* srcBuffer, const size_t srcSize, + void* dstBuffer, const size_t dstSize, + void* dictBuffer, const size_t dictSize, + const size_t* fileSizes, const size_t nbFiles, ZSTD_CCtx* ctx, ZSTD_DCtx* dctx, const ZSTD_compressionParameters cParams, const constraint_t target, BMK_result_t* winnerResult, U8* memoTable, - U32* varyParams, const int varyLen) { + const U32* varyParams, const int varyLen) { size_t memind = memoTableInd(&cParams, varyParams, varyLen); - //BMK_translateAdvancedParams(cParams); if(memoTable[memind] >= INFEASIBLE_THRESHOLD) { return INFEASIBLE_RESULT; //see feasibleBenchMemo for concerns } else { - int res = infeasibleBench(resultPtr, srcBuffer, srcSize, dstBuffer, dstSize, dictBuffer, dictSize, ctx, dctx, + int res = infeasibleBench(resultPtr, srcBuffer, srcSize, dstBuffer, dstSize, dictBuffer, dictSize, fileSizes, nbFiles, ctx, dctx, cParams, target, winnerResult); if(res == FEASIBLE_RESULT) { memoTable[memind] = 255; //infeasible resultscores could still be normal feasible. @@ -1432,14 +1440,18 @@ typedef int (*BMK_benchMemo_t)(BMK_result_t*, const void*, size_t, void*, size_t const ZSTD_compressionParameters, const constraint_t, BMK_result_t*, U8*, U32*, const int); //varArray should be sanitized when this is called. -//TODO: transition to simpler greedy method if evaluation time is too long? -//would it be better to start at best feasible via feasible or infeasible metric? both? //possibility climb is infeasible, responsibility of caller to check that. but if something feasible is evaluated, it will be returned // *actually if it performs too //sanitize all params here. //all generation after random should be sanitized. (maybe sanitize random) -static winnerInfo_t climbOnce(constraint_t target, U32* varArray, const int varLen, U8* memoTable, - const void* srcBuffer, size_t srcSize, void* dstBuffer, size_t dstSize, void* dictBuffer, size_t dictSize, ZSTD_CCtx* ctx, ZSTD_DCtx* dctx, ZSTD_compressionParameters init) { +static winnerInfo_t climbOnce(const constraint_t target, + const U32* varArray, const int varLen, U8* memoTable, + const void* srcBuffer, size_t srcSize, + void* dstBuffer, const size_t dstSize, + void* dictBuffer, const size_t dictSize, + const size_t* fileSizes, const size_t nbFiles, + ZSTD_CCtx* ctx, ZSTD_DCtx* dctx, + const ZSTD_compressionParameters init) { //pick later initializations non-randomly? high dist from explored nodes. //how to do this efficiently? (might not be too much of a problem, happens rarely, running time probably dominated by benchmarking) //distance maximizing selection? @@ -1459,28 +1471,20 @@ static winnerInfo_t climbOnce(constraint_t target, U32* varArray, const int varL /* ineasible -> (hopefully) feasible */ /* when nothing is found, this garbages part 2. */ { - //TODO: initialize these values! winnerInfo_t bestFeasible1; /* uses feasibleBench Metric */ - winnerInfo_t bestFeasible2; /* uses resultScore Metric */ //init these params bestFeasible1.params = cparam; - bestFeasible2.params = cparam; bestFeasible1.result.cSpeed = 0; bestFeasible1.result.dSpeed = 0; bestFeasible1.result.cMem = (size_t)-1; bestFeasible1.result.cSize = (size_t)-1; - bestFeasible2.result.cSpeed = 0; - bestFeasible2.result.dSpeed = 0; - bestFeasible2.result.cMem = (size_t)-1; - bestFeasible2.result.cSize = (size_t)-1; DISPLAY("Climb Part 1\n"); while(better) { - //UTIL_time_t timestart = UTIL_getTime(); TODO: adjust sampling based on time int i, d; better = 0; - DISPLAY("Start\n"); + DEBUGOUTPUT("Start\n"); cparam = winnerInfo.params; BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, srcSize); candidateInfo.params = cparam; @@ -1496,18 +1500,16 @@ static winnerInfo_t climbOnce(constraint_t target, U32* varArray, const int varL srcBuffer, srcSize, dstBuffer, dstSize, dictBuffer, dictSize, + fileSizes, nbFiles, ctx, dctx, candidateInfo.params, target, &winnerInfo.result, memoTable, varArray, varLen); if(res == FEASIBLE_RESULT) { /* synonymous with better when called w/ infeasibleBM */ winnerInfo = candidateInfo; - //BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, srcSize); + BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, srcSize); better = 1; - if(feasible(candidateInfo.result, target)) { - bestFeasible2 = winnerInfo; - if(objective_lt(bestFeasible1.result, bestFeasible2.result)) { - bestFeasible1 = bestFeasible2; /* using feasibleBench metric */ - } + if(feasible(candidateInfo.result, target) && objective_lt(bestFeasible1.result, winnerInfo.result)) { + bestFeasible1 = winnerInfo; } } } @@ -1521,18 +1523,16 @@ static winnerInfo_t climbOnce(constraint_t target, U32* varArray, const int varL srcBuffer, srcSize, dstBuffer, dstSize, dictBuffer, dictSize, + fileSizes, nbFiles, ctx, dctx, candidateInfo.params, target, &winnerInfo.result, memoTable, varArray, varLen); if(res == FEASIBLE_RESULT) { winnerInfo = candidateInfo; - //BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, srcSize); + BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, srcSize); better = 1; - if(feasible(candidateInfo.result, target)) { //TODO: maybe just break here and move on to part 2? - bestFeasible2 = winnerInfo; - if(objective_lt(bestFeasible1.result, bestFeasible2.result)) { - bestFeasible1 = bestFeasible2; - } + if(feasible(candidateInfo.result, target) && objective_lt(bestFeasible1.result, winnerInfo.result)) { + bestFeasible1 = winnerInfo; } } } @@ -1553,18 +1553,16 @@ static winnerInfo_t climbOnce(constraint_t target, U32* varArray, const int varL srcBuffer, srcSize, dstBuffer, dstSize, dictBuffer, dictSize, + fileSizes, nbFiles, ctx, dctx, candidateInfo.params, target, &winnerInfo.result, memoTable, varArray, varLen); if(res == FEASIBLE_RESULT) { /* synonymous with better in this case*/ winnerInfo = candidateInfo; - //BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, srcSize); + BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, srcSize); better = 1; - if(feasible(candidateInfo.result, target)) { - bestFeasible2 = winnerInfo; - if(objective_lt(bestFeasible1.result, bestFeasible2.result)) { - bestFeasible1 = bestFeasible2; - } + if(feasible(candidateInfo.result, target) && objective_lt(bestFeasible1.result, winnerInfo.result)) { + bestFeasible1 = winnerInfo; } } @@ -1576,15 +1574,12 @@ static winnerInfo_t climbOnce(constraint_t target, U32* varArray, const int varL //bias to test previous delta? //change cparam -> candidate before restart } - //TODO:Consider if this is best config. idea: explore from obj best keep rbest - cparam = bestFeasible2.params; - candidateInfo = bestFeasible2; winnerInfo = bestFeasible1; } - //is it better to break here instead of bumbling about? + //break out if no feasible. if(winnerInfo.result.cMem == (U32)-1) { - DISPLAY("No Feasible Found\n"); + DEBUGOUTPUT("No Feasible Found\n"); return winnerInfo; } DISPLAY("Climb Part 2\n"); @@ -1592,14 +1587,12 @@ static winnerInfo_t climbOnce(constraint_t target, U32* varArray, const int varL better = 1; /* feasible -> best feasible (hopefully) */ { - while(better) { - - //UTIL_time_t timestart = UTIL_getTime(); //TODO: if benchmarking is taking too long, be more greedy. + while(better) { int i, d; better = 0; BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, srcSize); //all dist-1 targets - cparam = winnerInfo.params; //TODO: this messes the taking bestFeasible1, bestFeasible2 + cparam = winnerInfo.params; candidateInfo.params = cparam; for(i = 0; i < varLen; i++) { paramVaryOnce(varArray[i], 1, &candidateInfo.params); @@ -1612,12 +1605,13 @@ static winnerInfo_t climbOnce(constraint_t target, U32* varArray, const int varL srcBuffer, srcSize, dstBuffer, dstSize, dictBuffer, dictSize, + fileSizes, nbFiles, ctx, dctx, candidateInfo.params, target, &winnerInfo.result, memoTable, varArray, varLen); if(res == FEASIBLE_RESULT) { winnerInfo = candidateInfo; - //BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, srcSize); + BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, srcSize); better = 1; } } @@ -1626,17 +1620,17 @@ static winnerInfo_t climbOnce(constraint_t target, U32* varArray, const int varL candidateInfo.params = sanitizeParams(candidateInfo.params); //evaluate if(!ZSTD_isError(ZSTD_checkCParams(candidateInfo.params))) { - //if(cParamValid(candidateInfo.params)) { int res = feasibleBenchMemo(&candidateInfo.result, srcBuffer, srcSize, dstBuffer, dstSize, dictBuffer, dictSize, + fileSizes, nbFiles, ctx, dctx, candidateInfo.params, target, &winnerInfo.result, memoTable, varArray, varLen); if(res == FEASIBLE_RESULT) { winnerInfo = candidateInfo; - //BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, srcSize); + BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, srcSize); better = 1; } } @@ -1653,12 +1647,13 @@ static winnerInfo_t climbOnce(constraint_t target, U32* varArray, const int varL srcBuffer, srcSize, dstBuffer, dstSize, dictBuffer, dictSize, + fileSizes, nbFiles, ctx, dctx, candidateInfo.params, target, &winnerInfo.result, memoTable, varArray, varLen); if(res == FEASIBLE_RESULT) { winnerInfo = candidateInfo; - //BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, srcSize); + BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, srcSize); better = 1; } } @@ -1684,29 +1679,26 @@ static winnerInfo_t climbOnce(constraint_t target, U32* varArray, const int varL //only real use for paramTarget is to get the fixed values, right? static winnerInfo_t optimizeFixedStrategy( const void* srcBuffer, const size_t srcSize, - void* dstBuffer, size_t dstSize, - void* dictBuffer, size_t dictSize, - constraint_t target, ZSTD_compressionParameters paramTarget, - ZSTD_strategy strat, U32* varArray, int varLen) { - int i = 0; //TODO: Temp fix 10 iters, check effects of changing this? + void* dstBuffer, const size_t dstSize, + void* dictBuffer, const size_t dictSize, + const size_t* fileSizes, const size_t nbFiles, + const constraint_t target, ZSTD_compressionParameters paramTarget, + const ZSTD_strategy strat, const U32* varArray, const int varLen, U8* memoTable) { + int i = 0; U32* varNew = malloc(sizeof(U32) * varLen); int varLenNew = sanitizeVarArray(varLen, varArray, varNew, strat); - size_t memoLen = memoTableLen(varNew, varLenNew); - U8* memoTable = malloc(sizeof(U8) * memoLen); ZSTD_compressionParameters init; ZSTD_CCtx* ctx = ZSTD_createCCtx(); ZSTD_DCtx* dctx = ZSTD_createDCtx(); winnerInfo_t winnerInfo, candidateInfo; winnerInfo.result.cSpeed = 0; winnerInfo.result.dSpeed = 0; - winnerInfo.result.cMem = (size_t)(-1); - winnerInfo.result.cSize = (size_t)(-1); + winnerInfo.result.cMem = (size_t)(-1LL); + winnerInfo.result.cSize = (size_t)(-1LL); /* so climb is given the right fixed strategy */ paramTarget.strategy = strat; /* to pass ZSTD_checkCParams */ - memoTableInit(memoTable, paramTarget, target, varNew, varLenNew, srcSize); - //needs to happen after memoTableInit as that assumes 0 = undefined. cParamZeroMin(¶mTarget); @@ -1718,11 +1710,11 @@ static winnerInfo_t optimizeFixedStrategy( goto _cleanUp; } - while(i < 10) { - DISPLAY("Restart\n"); - //TODO: look into improving this to maximize distance from searched infeasible stuff / towards promising regions? + while(i < 10) { //make i adjustable (user input?) depending on how much time they have. + DISPLAY("Restart\n"); //TODO: make better printing across restarts + //look into improving this to maximize distance from searched infeasible stuff / towards promising regions? randomConstrainedParams(&init, varNew, varLenNew, memoTable); - candidateInfo = climbOnce(target, varNew, varLenNew, memoTable, srcBuffer, srcSize, dstBuffer, dstSize, dictBuffer, dictSize, ctx, dctx, init); + candidateInfo = climbOnce(target, varNew, varLenNew, memoTable, srcBuffer, srcSize, dstBuffer, dstSize, dictBuffer, dictSize, fileSizes, nbFiles, ctx, dctx, init); if(objective_lt(winnerInfo.result, candidateInfo.result)) { winnerInfo = candidateInfo; DISPLAY("New Winner: "); @@ -1735,7 +1727,6 @@ static winnerInfo_t optimizeFixedStrategy( _cleanUp: ZSTD_freeCCtx(ctx); ZSTD_freeDCtx(dctx); - free(memoTable); free(varNew); return winnerInfo; } @@ -1777,20 +1768,54 @@ static int BMK_loadFiles(void* buffer, size_t bufferSize, fclose(f); } - if (totalSize == 0) { DISPLAY("no data to bench\n"); return 12; } + if (totalSize == 0) { DISPLAY("\nno data to bench\n"); return 12; } return 0; } -// bigger and (hopefully) better* than optimizeForSize -// TODO: allow accept multiple files like benchFiles or bench.c fn's -static int optimizeForSize2(const char* const * const fileNamesTable, const size_t nbFiles, const char* dictFileName, constraint_t target, ZSTD_compressionParameters paramTarget) +//goes best, best-1, best+1, best-2, ... +//return 0 if nothing remaining +static int nextStrategy(const int currentStrategy, const int bestStrategy) { + if(bestStrategy <= currentStrategy) { + int candidate = 2 * bestStrategy - currentStrategy - 1; + if(candidate < 1) { + candidate = currentStrategy + 1; + if(candidate > (int)ZSTD_btultra) { + return 0; + } else { + return candidate; + } + } else { + return candidate; + } + } else { /* bestStrategy >= currentStrategy */ + int candidate = 2 * bestStrategy - currentStrategy; + if(candidate > (int)ZSTD_btultra) { + candidate = currentStrategy - 1; + if(candidate < 1) { + return 0; + } else { + return candidate; + } + } else { + return candidate; + } + } +} + +//optimize fixed strategy. +static int optimizeForSize(const char* const * const fileNamesTable, const size_t nbFiles, const char* dictFileName, constraint_t target, ZSTD_compressionParameters paramTarget) { size_t benchedSize; - void* origBuff; - void* dictBuffer; - size_t dictBufferSize; + void* origBuff = NULL; + void* dictBuffer = NULL; + size_t dictBufferSize = 0; U32 varArray [NUM_PARAMS]; - int varLen = variableParams(paramTarget, varArray); + int ret = 0; + size_t* fileSizes = calloc(sizeof(size_t),nbFiles); + const int varLen = variableParams(paramTarget, varArray); + U8** allMT = NULL; + g_targetConstraints = target; + g_winner.result.cSize = (size_t)-1; /* Init */ if(!cParamValid(paramTarget)) { return 10; @@ -1801,20 +1826,24 @@ static int optimizeForSize2(const char* const * const fileNamesTable, const size U64 const dictFileSize = UTIL_getFileSize(dictFileName); if (dictFileSize > 64 MB) { DISPLAY("dictionary file %s too large", dictFileName); - return 10; + ret = 10; + goto _cleanUp; } dictBufferSize = (size_t)dictFileSize; dictBuffer = malloc(dictBufferSize); if (dictBuffer==NULL) { DISPLAY("not enough memory for dictionary (%u bytes)", (U32)dictBufferSize); - return 11; + ret = 11; + goto _cleanUp; + } + { int errorCode = BMK_loadFiles(dictBuffer, dictBufferSize, &dictBufferSize, &dictFileName, 1); if(errorCode) { - free(dictBuffer); - return errorCode; + ret = errorCode; + goto _cleanUp; } } } @@ -1823,41 +1852,46 @@ static int optimizeForSize2(const char* const * const fileNamesTable, const size if(nbFiles == 1) { DISPLAY("Loading %s... \r", fileNamesTable[0]); } else { - DISPLAY("Loading %zd Files... \r", nbFiles); + DISPLAY("Loading %lu Files... \r", (unsigned long)nbFiles); } { U64 const totalSizeToLoad = UTIL_getTotalFileSize(fileNamesTable, nbFiles); int ec; - size_t* fileSizes = calloc(sizeof(size_t),nbFiles); + unsigned i; benchedSize = BMK_findMaxMem(totalSizeToLoad * 3) / 3; + origBuff = malloc(benchedSize); if(!origBuff || !fileSizes) { DISPLAY("Not enough memory for stuff\n"); - free(origBuff); - free(fileSizes); - free(dictBuffer); - return 1; + ret = 1; + goto _cleanUp; } ec = BMK_loadFiles(origBuff, benchedSize, fileSizes, fileNamesTable, nbFiles); if(ec) { DISPLAY("Error Loading Files"); - free(origBuff); - free(fileSizes); - free(dictBuffer); - return ec; + ret = ec; + goto _cleanUp; } - free(fileSizes); + benchedSize = 0; + for(i = 0; i < nbFiles; i++) { + benchedSize += fileSizes[i]; + } + origBuff = realloc(origBuff, benchedSize); } - + allMT = memoTableInitAll(paramTarget, target, varArray, varLen, benchedSize); + if(!allMT) { + ret = 2; + goto _cleanUp; + } /* bench */ DISPLAY("\r%79s\r", ""); if(nbFiles == 1) { DISPLAY("optimizing for %s", fileNamesTable[0]); } else { - DISPLAY("optimizing for %zd Files", nbFiles); + DISPLAY("optimizing for %lu Files", (unsigned long)nbFiles); } if(target.cSpeed != 0) { DISPLAY(" - limit compression speed %u MB/s", target.cSpeed / 1000000); } if(target.dSpeed != 0) { DISPLAY(" - limit decompression speed %u MB/s", target.dSpeed / 1000000); } @@ -1868,7 +1902,7 @@ static int optimizeForSize2(const char* const * const fileNamesTable, const size { ZSTD_CCtx* const ctx = ZSTD_createCCtx(); ZSTD_DCtx* const dctx = ZSTD_createDCtx(); winnerInfo_t winner; - //BMK_result_t candidate; + U32 varNew[NUM_PARAMS]; const size_t blockSize = g_blockSize ? g_blockSize : benchedSize; U32 const maxNbBlocks = (U32) ((benchedSize + (blockSize-1)) / blockSize) + 1; const size_t maxCompressedSize = ZSTD_compressBound(benchedSize) + (maxNbBlocks * 1024); @@ -1877,49 +1911,120 @@ static int optimizeForSize2(const char* const * const fileNamesTable, const size /* init */ if (ctx==NULL) { DISPLAY("\n ZSTD_createCCtx error \n"); free(origBuff); return 14;} if(compressedBuffer==NULL) { DISPLAY("\n Allocation Error \n"); free(origBuff); free(ctx); return 15; } - memset(&winner, 0, sizeof(winner)); winner.result.cSize = (size_t)(-1); /* find best solution from default params */ - //Can't do this w/ cparameter constraints - //still useful though? - /* - { const int maxSeeds = g_noSeed ? 1 : ZSTD_maxCLevel(); - int i; - for (i=1; i<=maxSeeds; i++) { - ZSTD_compressionParameters const CParams = ZSTD_getCParams(i, blockSize, 0); - BMK_benchParam(&candidate, origBuff, benchedSize, ctx, dctx, CParams); - if (!feasible(candidate, target) ) { - break; + { + /* strategy selection */ + //TODO: don't factor memory into strategy selection in constraint_t + const int maxSeeds = g_noSeed ? 1 : ZSTD_maxCLevel(); + DEBUGOUTPUT("Strategy Selection\n"); + if(varLen == NUM_PARAMS && paramTarget.strategy == 0) { /* no variable based constraints */ + BMK_result_t candidate; + int feas = 0, i; + for (i=1; i<=maxSeeds; i++) { + ZSTD_compressionParameters const CParams = ZSTD_getCParams(i, blockSize, 0); + int ec = BMK_benchParam(&candidate, origBuff, benchedSize, fileSizes, nbFiles, ctx, dctx, CParams); + BMK_printWinner(stdout, i, candidate, CParams, benchedSize); + + if(!ec) { + if(feas) { + if(feasible(candidate, relaxTarget(target)) && objective_lt(winner.result, candidate)) { + winner.result = candidate; + winner.params = CParams; + } + } else { + if(feasible(candidate, relaxTarget(target))) { + feas = 1; + winner.result = candidate; + winner.params = CParams; + + } else { + if(resultScore(candidate, benchedSize, target) > resultScore(winner.result, benchedSize, target)) { + winner.result = candidate; + winner.params = CParams; + } + } + } + } + } //best, -1, +1, ..., + + } else if (paramTarget.strategy == 0) { //constrained + int feas = 0, i, j; + for(j = 1; j < 10; j++) { + for(i = 1; i <= maxSeeds; i++) { + int varLenNew = sanitizeVarArray(varLen, varArray, varNew, i); + ZSTD_compressionParameters candidateParams = paramTarget; + BMK_result_t candidate; + int ec; + randomConstrainedParams(&candidateParams, varNew, varLenNew, allMT[i]); + cParamZeroMin(&candidateParams); + candidateParams = sanitizeParams(candidateParams); + ec = BMK_benchParam(&candidate, origBuff, benchedSize, fileSizes, nbFiles, ctx, dctx, candidateParams); + + if(!ec) { + if(feas) { + if(feasible(candidate, relaxTarget(target)) && objective_lt(winner.result, candidate)) { + winner.result = candidate; + winner.params = candidateParams; + BMK_printWinner(stdout, i, winner.result, winner.params, benchedSize); + } + } else { + if(feasible(candidate, relaxTarget(target))) { + feas = 1; + winner.result = candidate; + winner.params = candidateParams; + BMK_printWinner(stdout, i, winner.result, winner.params, benchedSize); + + } else { + if(resultScore(candidate, benchedSize, target) > resultScore(winner.result, benchedSize, target)) { + winner.result = candidate; + winner.params = candidateParams; + BMK_printWinner(stdout, i, winner.result, winner.params, benchedSize); + } + } + } + } + + } } - if (feasible(candidate,target) && objective_lt(winner.result, candidate)) - { - winner.params = CParams; - winner.result = candidate; - BMK_printWinner(stdout, i, winner.result, winner.params, benchedSize); - } } - }*/ + } + } + BMK_printWinner(stdout, CUSTOM_LEVEL, winner.result, winner.params, benchedSize); - BMK_translateAdvancedParams(winner.params); - - /* start real tests */ + DEBUGOUTPUT("Real Opt\n"); + /* start 'real' tests */ { + int bestStrategy = (int)winner.params.strategy; if(paramTarget.strategy == 0) { - int st; - for(st = 1; st <= 8; st++) { - winnerInfo_t wc = optimizeFixedStrategy(origBuff, benchedSize, compressedBuffer, maxCompressedSize, dictBuffer, dictBufferSize, - target, paramTarget, st, varArray, varLen); - DISPLAY("StratNum %d\n", st); + int st = (int)winner.params.strategy; + + { + int varLenNew = sanitizeVarArray(varLen, varArray, varNew, st); + winnerInfo_t w1 = climbOnce(target, varNew, varLenNew, allMT[st], + origBuff, benchedSize, compressedBuffer, maxCompressedSize, dictBuffer, dictBufferSize, + fileSizes, nbFiles, ctx, dctx, winner.params); + if(objective_lt(winner.result, w1.result)) { + winner = w1; + } + } + + while(st) { + winnerInfo_t wc = optimizeFixedStrategy(origBuff, benchedSize, compressedBuffer, maxCompressedSize, dictBuffer, dictBufferSize, fileSizes, nbFiles, + target, paramTarget, st, varArray, varLen, allMT[st]); + DEBUGOUTPUT("StratNum %d\n", st); if(objective_lt(winner.result, wc.result)) { winner = wc; } + //TODO: we could double back to increase search of 'better' strategies + st = nextStrategy(st, bestStrategy); } } else { - winner = optimizeFixedStrategy(origBuff, benchedSize, compressedBuffer, maxCompressedSize, dictBuffer, dictBufferSize, - target, paramTarget, paramTarget.strategy, varArray, varLen); + winner = optimizeFixedStrategy(origBuff, benchedSize, compressedBuffer, maxCompressedSize, dictBuffer, dictBufferSize, fileSizes, nbFiles, + target, paramTarget, paramTarget.strategy, varArray, varLen, allMT[paramTarget.strategy]); } } @@ -1934,150 +2039,18 @@ static int optimizeForSize2(const char* const * const fileNamesTable, const size BMK_translateAdvancedParams(winner.params); DISPLAY("grillParams size - optimizer completed \n"); - /* clean up*/ - ZSTD_freeCCtx(ctx); - ZSTD_freeDCtx(dctx); - } - - free(origBuff); - return 0; -} - - -/* optimizeForSize(): - * targetSpeed : expressed in B/s */ -/* expresses targeted compression, decompression speeds and memory requirements */ -/* if state space is small (from paramTarget), exhaustive search? */ -//things to consider : if doing strategy-separate approach, what cutoffs to evaluate each strategy -//or do all? can't be absolute, should be relative after some sort of calibration -//(synthetic? test levels (we don't care about data specifics rn, scale?) ? -int optimizeForSize(const char* inFileName, constraint_t target, ZSTD_compressionParameters paramTarget) -{ - FILE* const inFile = fopen( inFileName, "rb" ); - U64 const inFileSize = UTIL_getFileSize(inFileName); - size_t benchedSize = BMK_findMaxMem(inFileSize*3) / 3; - void* origBuff; - U32 paramVarArray [NUM_PARAMS]; - int paramCount = variableParams(paramTarget, paramVarArray); - /* Init */ - if (inFile==NULL) { DISPLAY( "Pb opening %s\n", inFileName); return 11; } - if (inFileSize == UTIL_FILESIZE_UNKNOWN) { - DISPLAY("Pb evaluatin size of %s \n", inFileName); - fclose(inFile); - return 11; - } - - /* Memory allocation & restrictions */ - if ((U64)benchedSize > inFileSize) benchedSize = (size_t)inFileSize; - if (benchedSize < inFileSize) { - DISPLAY("Not enough memory for '%s' \n", inFileName); - fclose(inFile); - return 11; - } - - /* Alloc */ - origBuff = malloc(benchedSize); - if(!origBuff) { - DISPLAY("\nError: not enough memory!\n"); - fclose(inFile); - return 12; - } - - /* Fill input buffer */ - DISPLAY("Loading %s... \r", inFileName); - { size_t const readSize = fread(origBuff, 1, benchedSize, inFile); - fclose(inFile); - if(readSize != benchedSize) { - DISPLAY("\nError: problem reading file '%s' !! \n", inFileName); - free(origBuff); - return 13; - } } - - /* bench */ - DISPLAY("\r%79s\r", ""); - DISPLAY("optimizing for %s", inFileName); - if(target.cSpeed != 0) { DISPLAY(" - limit compression speed %u MB/s", target.cSpeed / 1000000); } - if(target.dSpeed != 0) { DISPLAY(" - limit decompression speed %u MB/s", target.dSpeed / 1000000); } - if(target.cMem != 0) { DISPLAY(" - limit memory %u MB", target.cMem / 1000000); } - DISPLAY("\n"); - { ZSTD_CCtx* const ctx = ZSTD_createCCtx(); - ZSTD_DCtx* const dctx = ZSTD_createDCtx(); - winnerInfo_t winner; - BMK_result_t candidate; - const size_t blockSize = g_blockSize ? g_blockSize : benchedSize; - - /* init */ - if (ctx==NULL) { DISPLAY("\n ZSTD_createCCtx error \n"); free(origBuff); return 14; } - - memset(&winner, 0, sizeof(winner)); - winner.result.cSize = (size_t)(-1); - - /* find best solution from default params */ - //Can't do this w/ cparameter constraints - { const int maxSeeds = g_noSeed ? 1 : ZSTD_maxCLevel(); - int i; - for (i=1; i<=maxSeeds; i++) { - ZSTD_compressionParameters const CParams = ZSTD_getCParams(i, blockSize, 0); - BMK_benchParam(&candidate, origBuff, benchedSize, ctx, dctx, CParams); - if (!feasible(candidate, target) ) { - break; - } - if (feasible(candidate,target) && objective_lt(winner.result, candidate)) - { - winner.params = CParams; - winner.result = candidate; - BMK_printWinner(stdout, i, winner.result, winner.params, benchedSize); - } } - } - BMK_printWinner(stdout, CUSTOM_LEVEL, winner.result, winner.params, benchedSize); - - BMK_translateAdvancedParams(winner.params); - - /* start tests */ - { time_t const grillStart = time(NULL); - do { - ZSTD_compressionParameters params = winner.params; - BYTE* b; - paramVariation(¶ms, paramVarArray, paramCount, 4); - if ((FUZ_rand(&g_rand) & 31) == 3) params = randomParams(); /* totally random config to improve search space */ - params = ZSTD_adjustCParams(params, blockSize, 0); - - /* exclude faster if already played set of params */ - if (FUZ_rand(&g_rand) & ((1 << *NB_TESTS_PLAYED(params))-1)) continue; - - /* test */ - b = NB_TESTS_PLAYED(params); - (*b)++; - BMK_benchParam(&candidate, origBuff, benchedSize, ctx, dctx, params); - - /* improvement found => new winner */ - if (feasible(candidate,target) && objective_lt(winner.result, candidate)) - { - winner.params = params; - winner.result = candidate; - BMK_printWinner(stdout, CUSTOM_LEVEL, winner.result, winner.params, benchedSize); - BMK_translateAdvancedParams(winner.params); - } - } while (BMK_timeSpan(grillStart) < g_grillDuration_s); - } - - /* no solution found */ - if(winner.result.cSize == (size_t)-1) { - DISPLAY("No feasible solution found\n"); - return 1; - } - /* end summary */ - BMK_printWinner(stdout, CUSTOM_LEVEL, winner.result, winner.params, benchedSize); - BMK_translateAdvancedParams(winner.params); - DISPLAY("grillParams size - optimizer completed \n"); /* clean up*/ ZSTD_freeCCtx(ctx); ZSTD_freeDCtx(dctx); - } + } +_cleanUp: + free(fileSizes); + free(dictBuffer); + memoTableFreeAll(allMT); free(origBuff); - return 0; + return ret; } static void errorOut(const char* msg) @@ -2136,6 +2109,7 @@ static int usage_advanced(void) DISPLAY( " -P# : generated sample compressibility (default : %.1f%%) \n", COMPRESSIBILITY_DEFAULT * 100); DISPLAY( " -t# : Caps runtime of operation in seconds (default : %u seconds (%.1f hours)) \n", (U32)g_grillDuration_s, g_grillDuration_s / 3600); DISPLAY( " -v : Prints Benchmarking output\n"); + DISPLAY( " -D : Next argument dictionary file\n"); return 0; } @@ -2163,6 +2137,8 @@ int main(int argc, const char** argv) assert(argc>=1); /* for exename */ + g_time = UTIL_getTime(); + /* Welcome message */ DISPLAY(WELCOME_MESSAGE); @@ -2397,7 +2373,7 @@ int main(int argc, const char** argv) } } else { if (optimizer) { - result = optimizeForSize2(argv+filenamesStart, argc-filenamesStart, dictFileName, target, paramTarget); + result = optimizeForSize(argv+filenamesStart, argc-filenamesStart, dictFileName, target, paramTarget); } else { result = benchFiles(argv+filenamesStart, argc-filenamesStart); } } From 7b5b3d7ae383f4efe53a95d68a4a1f1760f78650 Mon Sep 17 00:00:00 2001 From: George Lu Date: Mon, 16 Jul 2018 16:16:31 -0700 Subject: [PATCH 115/372] BenchMem with block compressed sizes passed back up --- programs/bench.c | 31 +-- programs/bench.h | 13 +- tests/paramgrill.c | 625 +++++++++++++++++++++++++++++++++++++-------- 3 files changed, 540 insertions(+), 129 deletions(-) diff --git a/programs/bench.c b/programs/bench.c index f5184fa8f..8e57db4c9 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -188,7 +188,7 @@ static void BMK_initCCtx(ZSTD_CCtx* ctx, ZSTD_CCtx_setParameter(ctx, ZSTD_p_searchLog, comprParams->searchLog); ZSTD_CCtx_setParameter(ctx, ZSTD_p_minMatch, comprParams->searchLength); ZSTD_CCtx_setParameter(ctx, ZSTD_p_targetLength, comprParams->targetLength); - ZSTD_CCtx_setParameter(ctx, ZSTD_p_compressionStrategy , comprParams->strategy); + ZSTD_CCtx_setParameter(ctx, ZSTD_p_compressionStrategy, comprParams->strategy); ZSTD_CCtx_loadDictionary(ctx, dictBuffer, dictBufferSize); } @@ -281,8 +281,6 @@ static size_t local_defaultDecompress( } -volatile char g_touched; - /* initFn will be measured once, bench fn will be measured x times */ /* benchFn should return error value or out Size */ /* takes # of blocks and list of size & stuff for each. */ @@ -293,7 +291,7 @@ BMK_customReturn_t BMK_benchFunction( BMK_initFn_t initFn, void* initPayload, size_t blockCount, const void* const * const srcBlockBuffers, const size_t* srcBlockSizes, - void** const dstBlockBuffers, size_t* dstBlockCapacities, + void* const * const dstBlockBuffers, size_t* dstBlockCapacitiesToSizes, unsigned nbLoops) { size_t srcSize = 0, dstSize = 0, ind = 0; U64 totalTime; @@ -310,20 +308,13 @@ BMK_customReturn_t BMK_benchFunction( } { - - unsigned i, j; + size_t i; for(i = 0; i < blockCount; i++) { - for(j = 0; j < srcBlockSizes[i]; j++) { - g_touched = ((const char*)srcBlockBuffers[i])[j]; /* touch */ - } - } - for(i = 0; i < blockCount; i++) { - memset(dstBlockBuffers[i], 0xE5, dstBlockCapacities[i]); /* warm up and erase result buffer */ + memset(dstBlockBuffers[i], 0xE5, dstBlockCapacitiesToSizes[i]); /* warm up and erase result buffer */ } //UTIL_sleepMilli(5); /* give processor time to other processes */ //UTIL_waitForNextTick(); - } { @@ -332,17 +323,13 @@ BMK_customReturn_t BMK_benchFunction( if(initFn != NULL) { initFn(initPayload); } for(i = 0; i < nbLoops; i++) { for(j = 0; j < blockCount; j++) { - size_t res = benchFn(srcBlockBuffers[j], srcBlockSizes[j], dstBlockBuffers[j], dstBlockCapacities[j], benchPayload); + size_t res = benchFn(srcBlockBuffers[j], srcBlockSizes[j], dstBlockBuffers[j], dstBlockCapacitiesToSizes[j], benchPayload); if(ZSTD_isError(res)) { EXM_THROW_ND(2, BMK_customReturn_t, "Function benchmarking failed on block %u of size %u : %s \n", - j, (U32)dstBlockCapacities[j], ZSTD_getErrorName(res)); + j, (U32)dstBlockCapacitiesToSizes[j], ZSTD_getErrorName(res)); } else if(firstIter) { dstSize += res; - //Make compressed blocks continuous - if(j != blockCount - 1) { - dstBlockBuffers[j+1] = (void*)((char*)dstBlockBuffers[j] + res); - dstBlockCapacities[j] = res; - } + dstBlockCapacitiesToSizes[j] = res; } } firstIter = 0; @@ -382,7 +369,7 @@ BMK_customTimedReturn_t BMK_benchFunctionTimed( BMK_initFn_t initFn, void* initPayload, size_t blockCount, const void* const* const srcBlockBuffers, const size_t* srcBlockSizes, - void** const dstBlockBuffers, size_t* dstBlockCapacities) + void * const * const dstBlockBuffers, size_t * dstBlockCapacitiesToSizes) { U64 fastest = cont->fastestTime; int completed = 0; @@ -399,7 +386,7 @@ BMK_customTimedReturn_t BMK_benchFunctionTimed( } r.result = BMK_benchFunction(benchFn, benchPayload, initFn, initPayload, - blockCount, srcBlockBuffers, srcBlockSizes, dstBlockBuffers, dstBlockCapacities, cont->nbLoops); + blockCount, srcBlockBuffers, srcBlockSizes, dstBlockBuffers, dstBlockCapacitiesToSizes, cont->nbLoops); if(r.result.error) { /* completed w/ error */ r.completed = 1; return r; diff --git a/programs/bench.h b/programs/bench.h index 1a298cc48..25d9f24a7 100644 --- a/programs/bench.h +++ b/programs/bench.h @@ -122,8 +122,6 @@ BMK_return_t BMK_syntheticTest(int cLevel, double compressibility, * (cLevel, comprParams + adv in advanced Mode) */ /* srcBuffer - data source, expected to be valid compressed data if in Decode Only Mode * srcSize - size of data in srcBuffer - * dstBuffer - destination buffer to write compressed output in, optional (NULL) - * dstCapacity - capacity of destination buffer, give 0 if dstBuffer = NULL * cLevel - compression level * comprParams - basic compression parameters * dictBuffer - a dictionary if used, null otherwise @@ -144,7 +142,10 @@ BMK_return_t BMK_benchMem(const void* srcBuffer, size_t srcSize, ZSTD_CCtx* ctx, ZSTD_DCtx* dctx, int displayLevel, const char* displayName); -/* See benchMem for normal parameter uses and return, see advancedParams_t for adv */ +/* See benchMem for normal parameter uses and return, see advancedParams_t for adv + * dstBuffer - destination buffer to write compressed output in, NULL if none provided. + * dstCapacity - capacity of destination buffer, give 0 if dstBuffer = NULL + */ BMK_return_t BMK_benchMemAdvanced(const void* srcBuffer, size_t srcSize, void* dstBuffer, size_t dstCapacity, const size_t* fileSizes, unsigned nbFiles, @@ -174,7 +175,7 @@ typedef size_t (*BMK_initFn_t)(void*); * srcBuffers - an array of buffers to be operated on by benchFn * srcSizes - an array of the sizes of above buffers * dstBuffers - an array of buffers to be written into by benchFn - * dstCapacities - an array of the capacities of above buffers. + * dstCapacitiesToSizes - an array of the capacities of above buffers. Output modified to compressed sizes of those blocks. * nbLoops - defines number of times benchFn is run. * return * .error will give a nonzero value if ZSTD_isError() is nonzero for any of the return @@ -191,7 +192,7 @@ BMK_customReturn_t BMK_benchFunction( BMK_initFn_t initFn, void* initPayload, size_t blockCount, const void* const * const srcBuffers, const size_t* srcSizes, - void** const dstBuffers, size_t* dstCapacities, + void * const * const dstBuffers, size_t* dstCapacitiesToSizes, unsigned nbLoops); @@ -220,7 +221,7 @@ BMK_customTimedReturn_t BMK_benchFunctionTimed(BMK_timedFnState_t* cont, BMK_initFn_t initFn, void* initPayload, size_t blockCount, const void* const * const srcBlockBuffers, const size_t* srcBlockSizes, - void** const dstBlockBuffers, size_t* dstBlockCapacities); + void* const * const dstBlockBuffers, size_t* dstBlockCapacitiesToSizes); #endif /* BENCH_H_121279284357 */ diff --git a/tests/paramgrill.c b/tests/paramgrill.c index 9af60a740..e4d468846 100644 --- a/tests/paramgrill.c +++ b/tests/paramgrill.c @@ -27,7 +27,8 @@ #include "xxhash.h" #include "util.h" #include "bench.h" - +#include "zstd_errors.h" +#include "zstd_internal.h" /*-************************************ * Constants @@ -36,11 +37,6 @@ #define AUTHOR "Yann Collet" #define WELCOME_MESSAGE "*** %s %s %i-bits, by %s ***\n", PROGRAM_DESCRIPTION, ZSTD_VERSION_STRING, (int)(sizeof(void*)*8), AUTHOR - -#define KB *(1<<10) -#define MB *(1<<20) -#define GB *(1ULL<<30) - #define TIMELOOP_NANOSEC (1*1000000000ULL) /* 1 second */ #define NBLOOPS 2 @@ -240,7 +236,6 @@ static int cParamValid(ZSTD_compressionParameters paramTarget) { return 1; } -//TODO: let targetLength = 0; static void cParamZeroMin(ZSTD_compressionParameters* paramTarget) { paramTarget->windowLog = paramTarget->windowLog ? paramTarget->windowLog : ZSTD_WINDOWLOG_MIN; paramTarget->searchLog = paramTarget->searchLog ? paramTarget->searchLog : ZSTD_SEARCHLOG_MIN; @@ -261,7 +256,7 @@ static int feasible(const BMK_result_t results, const constraint_t target) { return (results.cSpeed >= target.cSpeed) && (results.dSpeed >= target.dSpeed) && (results.cMem <= target.cMem || !target.cMem); } -#define EPSILON 0.01 +#define EPSILON 0.001 static int epsilonEqual(const double c1, const double c2) { return MAX(c1/c2,c2/c1) < 1 + EPSILON; } @@ -274,8 +269,8 @@ static int eqZero(const double c1) { /* returns 1 if result2 is strictly 'better' than result1 */ /* strict comparison / cutoff based */ static int objective_lt(const BMK_result_t result1, const BMK_result_t result2) { - return (result1.cSize > result2.cSize) || (epsilonEqual(result1.cSize, result2.cSize) && result2.cSpeed > result1.cSpeed) - || (epsilonEqual(result1.cSize,result2.cSize) && epsilonEqual(result2.cSpeed, result1.cSpeed) && result2.dSpeed > result1.dSpeed); + return (result1.cSize > result2.cSize) || (result1.cSize == result2.cSize && result2.cSpeed > result1.cSpeed) + || (result1.cSize == result2.cSize && epsilonEqual(result2.cSpeed, result1.cSpeed) && result2.dSpeed > result1.dSpeed); } /* hill climbing value for part 1 */ @@ -352,9 +347,362 @@ typedef struct { ZSTD_compressionParameters params; } winnerInfo_t; +/*-******************************************************* +* From Paramgrill +*********************************************************/ + +static void BMK_initCCtx(ZSTD_CCtx* ctx, + const void* dictBuffer, size_t dictBufferSize, int cLevel, + const ZSTD_compressionParameters* comprParams, const BMK_advancedParams_t* adv) { + if (adv->nbWorkers==1) { + ZSTD_CCtx_setParameter(ctx, ZSTD_p_nbWorkers, 0); + } else { + ZSTD_CCtx_setParameter(ctx, ZSTD_p_nbWorkers, adv->nbWorkers); + } + ZSTD_CCtx_setParameter(ctx, ZSTD_p_compressionLevel, cLevel); + ZSTD_CCtx_setParameter(ctx, ZSTD_p_enableLongDistanceMatching, adv->ldmFlag); + ZSTD_CCtx_setParameter(ctx, ZSTD_p_ldmMinMatch, adv->ldmMinMatch); + ZSTD_CCtx_setParameter(ctx, ZSTD_p_ldmHashLog, adv->ldmHashLog); + ZSTD_CCtx_setParameter(ctx, ZSTD_p_ldmBucketSizeLog, adv->ldmBucketSizeLog); + ZSTD_CCtx_setParameter(ctx, ZSTD_p_ldmHashEveryLog, adv->ldmHashEveryLog); + ZSTD_CCtx_setParameter(ctx, ZSTD_p_windowLog, comprParams->windowLog); + ZSTD_CCtx_setParameter(ctx, ZSTD_p_hashLog, comprParams->hashLog); + ZSTD_CCtx_setParameter(ctx, ZSTD_p_chainLog, comprParams->chainLog); + ZSTD_CCtx_setParameter(ctx, ZSTD_p_searchLog, comprParams->searchLog); + ZSTD_CCtx_setParameter(ctx, ZSTD_p_minMatch, comprParams->searchLength); + ZSTD_CCtx_setParameter(ctx, ZSTD_p_targetLength, comprParams->targetLength); + ZSTD_CCtx_setParameter(ctx, ZSTD_p_compressionStrategy, comprParams->strategy); + ZSTD_CCtx_loadDictionary(ctx, dictBuffer, dictBufferSize); +} + + +static void BMK_initDCtx(ZSTD_DCtx* dctx, + const void* dictBuffer, size_t dictBufferSize) { + ZSTD_DCtx_loadDictionary(dctx, dictBuffer, dictBufferSize); +} + +typedef struct { + ZSTD_CCtx* ctx; + const void* dictBuffer; + size_t dictBufferSize; + int cLevel; + const ZSTD_compressionParameters* comprParams; + const BMK_advancedParams_t* adv; +} BMK_initCCtxArgs; + +static size_t local_initCCtx(void* payload) { + BMK_initCCtxArgs* ag = (BMK_initCCtxArgs*)payload; + BMK_initCCtx(ag->ctx, ag->dictBuffer, ag->dictBufferSize, ag->cLevel, ag->comprParams, ag->adv); + return 0; +} + +typedef struct { + ZSTD_DCtx* dctx; + const void* dictBuffer; + size_t dictBufferSize; +} BMK_initDCtxArgs; + +static size_t local_initDCtx(void* payload) { + BMK_initDCtxArgs* ag = (BMK_initDCtxArgs*)payload; + BMK_initDCtx(ag->dctx, ag->dictBuffer, ag->dictBufferSize); + return 0; +} + +/* additional argument is just the context */ +static size_t local_defaultCompress( + const void* srcBuffer, size_t srcSize, + void* dstBuffer, size_t dstSize, + void* addArgs) { + size_t moreToFlush = 1; + ZSTD_CCtx* ctx = (ZSTD_CCtx*)addArgs; + ZSTD_inBuffer in; + ZSTD_outBuffer out; + in.src = srcBuffer; + in.size = srcSize; + in.pos = 0; + out.dst = dstBuffer; + out.size = dstSize; + out.pos = 0; + while (moreToFlush) { + if(out.pos == out.size) { + return (size_t)-ZSTD_error_dstSize_tooSmall; + } + moreToFlush = ZSTD_compress_generic(ctx, &out, &in, ZSTD_e_end); + if (ZSTD_isError(moreToFlush)) { + return moreToFlush; + } + } + return out.pos; +} + +/* additional argument is just the context */ +static size_t local_defaultDecompress( + const void* srcBuffer, size_t srcSize, + void* dstBuffer, size_t dstSize, + void* addArgs) { + size_t moreToFlush = 1; + ZSTD_DCtx* dctx = (ZSTD_DCtx*)addArgs; + ZSTD_inBuffer in; + ZSTD_outBuffer out; + in.src = srcBuffer; + in.size = srcSize; + in.pos = 0; + out.dst = dstBuffer; + out.size = dstSize; + out.pos = 0; + while (moreToFlush) { + if(out.pos == out.size) { + return (size_t)-ZSTD_error_dstSize_tooSmall; + } + moreToFlush = ZSTD_decompress_generic(dctx, + &out, &in); + if (ZSTD_isError(moreToFlush)) { + return moreToFlush; + } + } + return out.pos; + +} + +/*-******************************************************* +* From Paramgrill End +*********************************************************/ + +/* Replicate function of benchMemAdvanced, but with pre-split src / dst buffers, with relevant info to invert it (compressedSizes) passed out. */ +/*BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, fileSizes, nbFiles, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); */ +/* nbSeconds used in same way as in BMK_advancedParams_t, as nbIters when in iterMode */ + +/* if in decodeOnly, then srcPtr's will be compressed blocks, and uncompressedBlocks will be written to dstPtrs? */ +/* dictionary nullable, nothing else though. */ +static BMK_return_t BMK_benchMemInvertible(const void * const * const srcPtrs, size_t const * const srcSizes, + void** dstPtrs, size_t* dstCapacityToSizes, U32 const nbBlocks, + const int cLevel, const ZSTD_compressionParameters* comprParams, + const void* dictBuffer, const size_t dictBufferSize, + ZSTD_CCtx* ctx, ZSTD_DCtx* dctx, + const BMK_mode_t mode, const BMK_loopMode_t loopMode, const unsigned nbSeconds) { + U32 i; + BMK_return_t results = { { 0, 0., 0., 0 }, 0 } ; + size_t srcSize = 0; + void** const resPtrs = malloc(sizeof(void*) * nbBlocks); /* only really needed in both mode. */ + size_t* const resSizes = malloc(sizeof(size_t) * nbBlocks); + int freeDST = 0; + + BMK_advancedParams_t adv = BMK_initAdvancedParams(); + adv.mode = mode; + adv.loopMode = loopMode; + adv.nbSeconds = nbSeconds; + + /* resSizes == srcSizes, but modifiable */ + memcpy(resSizes, srcSizes, sizeof(size_t) * nbBlocks); + + for(i = 0; i < nbBlocks; i++) { + srcSize += srcSizes[i]; + } + + if(!ctx || !dctx || !srcPtrs || ! srcSizes) + { + results.error = 31; + DISPLAY("error: passed in null argument\n"); + free(resPtrs); + free(resSizes); + return results; + } + if(!resPtrs || !resSizes) { + results.error = 32; + DISPLAY("error: allocation failed\n"); + free(resPtrs); + free(resSizes); + return results; + } + + /* so resPtr is continuous */ + resPtrs[0] = malloc(srcSize); + + if(!(resPtrs[0])) { + results.error = 32; + DISPLAY("error: allocation failed\n"); + free(resPtrs); + free(resSizes); + return results; + } + + for(i = 1; i < nbBlocks; i++) { + resPtrs[i] = (void*)(((char*)resPtrs[i-1]) + srcSizes[i-1]); + } + + /* allocate own dst if NULL */ + if(dstPtrs == NULL) { + freeDST = 1; + dstPtrs = malloc(nbBlocks * sizeof(void*)); + dstCapacityToSizes = malloc(nbBlocks * sizeof(size_t)); + if(dstPtrs == NULL) { + results.error = 33; + DISPLAY("error: allocation failed\n"); + free(resPtrs); + free(resSizes); + return results; + } + + if(mode == BMK_decodeOnly) { //dst is src + size_t dstSize = 0; + for(i = 0; i < nbBlocks; i++) { + dstCapacityToSizes[i] = ZSTD_getDecompressedSize(srcPtrs[i], srcSizes[i]); + dstSize += dstCapacityToSizes[i]; + } + dstPtrs[0] = malloc(dstSize); + if(dstPtrs[0] == NULL) { + results.error = 34; + DISPLAY("error: allocation failed\n"); + goto _cleanUp; + } + for(i = 1; i < nbBlocks; i++) { + dstPtrs[i] = (void*)(((char*)dstPtrs[i-1]) + ZSTD_getDecompressedSize(srcPtrs[i-1], srcSizes[i-1])); + } + } else { + dstPtrs[0] = malloc(ZSTD_compressBound(srcSize) + (nbBlocks * 1024)); + if(dstPtrs[0] == NULL) { + results.error = 35; + DISPLAY("error: allocation failed\n"); + goto _cleanUp; + } + dstCapacityToSizes[0] = ZSTD_compressBound(srcSizes[0]); + for(i = 1; i < nbBlocks; i++) { + dstPtrs[i] = (void*)(((char*)dstPtrs[i-1]) + dstCapacityToSizes[i-1]); + dstCapacityToSizes[i] = ZSTD_compressBound(srcSizes[i]); + } + } + } + + /* warmimg up memory */ + for(i = 0; i < nbBlocks; i++) { + RDG_genBuffer(dstPtrs[i], dstCapacityToSizes[i], 0.10, 0.50, 1); + } + + /* Bench */ + { + { + BMK_initCCtxArgs cctxprep; + BMK_initDCtxArgs dctxprep; + cctxprep.ctx = ctx; + cctxprep.dictBuffer = dictBuffer; + cctxprep.dictBufferSize = dictBufferSize; + cctxprep.cLevel = cLevel; + cctxprep.comprParams = comprParams; + cctxprep.adv = &adv; + dctxprep.dctx = dctx; + dctxprep.dictBuffer = dictBuffer; + dctxprep.dictBufferSize = dictBufferSize; + if(loopMode == BMK_timeMode) { + BMK_customTimedReturn_t intermediateResultCompress; + BMK_customTimedReturn_t intermediateResultDecompress; + BMK_timedFnState_t* timeStateCompress = BMK_createTimeState(nbSeconds); + BMK_timedFnState_t* timeStateDecompress = BMK_createTimeState(nbSeconds); + if(mode == BMK_compressOnly) { + intermediateResultCompress.completed = 0; + intermediateResultDecompress.completed = 1; + } else if (mode == BMK_decodeOnly) { + intermediateResultCompress.completed = 1; + intermediateResultDecompress.completed = 0; + } else { /* both */ + intermediateResultCompress.completed = 0; + intermediateResultDecompress.completed = 0; + } + while(!(intermediateResultCompress.completed && intermediateResultDecompress.completed)) { + if(!intermediateResultCompress.completed) { + intermediateResultCompress = BMK_benchFunctionTimed(timeStateCompress, &local_defaultCompress, (void*)ctx, &local_initCCtx, (void*)&cctxprep, + nbBlocks, srcPtrs, srcSizes, dstPtrs, dstCapacityToSizes); + if(intermediateResultCompress.result.error) { + results.error = intermediateResultCompress.result.error; + BMK_freeTimeState(timeStateCompress); + BMK_freeTimeState(timeStateDecompress); + goto _cleanUp; + } + results.result.cSpeed = ((double)srcSize / intermediateResultCompress.result.result.nanoSecPerRun) * 1000000000; + results.result.cSize = intermediateResultCompress.result.result.sumOfReturn; + } + + if(!intermediateResultDecompress.completed) { + if(mode == BMK_decodeOnly) { + intermediateResultDecompress = BMK_benchFunctionTimed(timeStateDecompress, &local_defaultDecompress, (void*)(dctx), &local_initDCtx, (void*)&dctxprep, + nbBlocks, (const void* const*)srcPtrs, srcSizes, dstPtrs, dstCapacityToSizes); + } else { /* both, decompressed result already written to dstPtr */ + intermediateResultDecompress = BMK_benchFunctionTimed(timeStateDecompress, &local_defaultDecompress, (void*)(dctx), &local_initDCtx, (void*)&dctxprep, + nbBlocks, (const void* const*)dstPtrs, dstCapacityToSizes, resPtrs, resSizes); + } + + if(intermediateResultDecompress.result.error) { + results.error = intermediateResultDecompress.result.error; + BMK_freeTimeState(timeStateCompress); + BMK_freeTimeState(timeStateDecompress); + goto _cleanUp; + } + results.result.dSpeed = ((double)srcSize / intermediateResultDecompress.result.result.nanoSecPerRun) * 1000000000; + } + } + BMK_freeTimeState(timeStateCompress); + BMK_freeTimeState(timeStateDecompress); + } else { //iterMode; + if(mode != BMK_decodeOnly) { + + BMK_customReturn_t compressionResults = BMK_benchFunction(&local_defaultCompress, (void*)ctx, &local_initCCtx, (void*)&cctxprep, + nbBlocks, srcPtrs, srcSizes, dstPtrs, dstCapacityToSizes, nbSeconds); + if(compressionResults.error) { + results.error = compressionResults.error; + goto _cleanUp; + } + if(compressionResults.result.nanoSecPerRun == 0) { + results.result.cSpeed = 0; + } else { + results.result.cSpeed = (double)srcSize / compressionResults.result.nanoSecPerRun * TIMELOOP_NANOSEC; + } + results.result.cSize = compressionResults.result.sumOfReturn; + } + if(mode != BMK_compressOnly) { + BMK_customReturn_t decompressionResults; + if(mode == BMK_decodeOnly) { + decompressionResults = BMK_benchFunction( + &local_defaultDecompress, (void*)(dctx), + &local_initDCtx, (void*)&dctxprep, nbBlocks, + (const void* const*)srcPtrs, srcSizes, dstPtrs, dstCapacityToSizes, + nbSeconds); + } else { + decompressionResults = BMK_benchFunction( + &local_defaultDecompress, (void*)(dctx), + &local_initDCtx, (void*)&dctxprep, nbBlocks, + (const void* const*)dstPtrs, dstCapacityToSizes, resPtrs, resSizes, + nbSeconds); + } + + if(decompressionResults.error) { + results.error = decompressionResults.error; + goto _cleanUp; + } + + if(decompressionResults.result.nanoSecPerRun == 0) { + results.result.dSpeed = 0; + } else { + results.result.dSpeed = (double)srcSize / decompressionResults.result.nanoSecPerRun * TIMELOOP_NANOSEC; + } + } + } + } + } /* Bench */ + results.result.cMem = (1 << (comprParams->windowLog)) + ZSTD_sizeof_CCtx(ctx); + +_cleanUp: + free(resPtrs[0]); + free(resPtrs); + free(resSizes); + if(freeDST) { + free(dstPtrs[0]); + free(dstPtrs); + } + return results; +} + /* global winner used for display. */ //Should be totally 0 initialized? -static winnerInfo_t g_winner; //TODO: ratio is infinite at initialization, instead of 0 +static winnerInfo_t g_winner; static constraint_t g_targetConstraints; static void BMK_printWinner(FILE* f, const U32 cLevel, const BMK_result_t result, const ZSTD_compressionParameters params, const size_t srcSize) @@ -379,7 +727,7 @@ static void BMK_printWinner(FILE* f, const U32 cLevel, const BMK_result_t result "/* %s */ /* R:%5.3f at %5.1f MB/s - %5.1f MB/s */", lvlstr, (double)srcSize / result.cSize, result.cSpeed / 1000000., result.dSpeed / 1000000.); - if(TIMED) { fprintf(f, " - %lu:%lu:%05.2f", (unsigned long) minutes / 60,(unsigned long) minutes % 60, (double)(time - minutes * TIMELOOP_NANOSEC * 60ULL)/TIMELOOP_NANOSEC); } + if(TIMED) { fprintf(f, " - %1lu:%2lu:%05.2f", (unsigned long) minutes / 60,(unsigned long) minutes % 60, (double)(time - minutes * TIMELOOP_NANOSEC * 60ULL)/TIMELOOP_NANOSEC); } fprintf(f, "\n"); if(objective_lt(g_winner.result, result) && feasible(result, g_targetConstraints)) { BMK_translateAdvancedParams(params); @@ -670,17 +1018,10 @@ static size_t memoTableLen(const U32* varyParams, const int varyLen) { //sort of ~lg2 (replace 1024 w/ 999, and add 0 at lower end of range) for memoTableInd Tlen static unsigned lg2(unsigned x) { - unsigned j = 1; if(x == 999) { return 11; } - if(!x) { - return 0; - } - while(x >>= 1) { - j++; - } - return j; + return x ? ZSTD_highbit32(x) + 1 : 0; } /* returns unique index of compression parameters */ @@ -729,7 +1070,7 @@ static void memoTableInit(U8* memoTable, ZSTD_compressionParameters paramConstra for(i = 0; i < arrayLen; i++) { memoTableIndInv(¶mConstraints, varyParams, varyLen, i); - if((ZSTD_estimateCCtxSize_usingCParams(paramConstraints) + (1ULL << paramConstraints.windowLog)) > (size_t)target.cMem + (size_t)(target.cMem / 10)) { + if(ZSTD_estimateCStreamSize_usingCParams(paramConstraints) > (size_t)target.cMem) { memoTable[i] = 255; j++; } @@ -765,8 +1106,6 @@ static void memoTableInit(U8* memoTable, ZSTD_compressionParameters paramConstra /* inits memotables for all (including mallocs), all strategies */ /* takes unsanitized varyParams */ - -//TODO: check for errors/nulls static U8** memoTableInitAll(ZSTD_compressionParameters paramConstraints, constraint_t target, const U32* varyParams, const int varyLen, const size_t srcSize) { U32 varNew[NUM_PARAMS]; int varLenNew; @@ -1099,28 +1438,29 @@ static int uncertainComparison(double const uncertaintyConstantC, double const u #define FEASIBLE_RESULT 1 #define ERROR_RESULT 2 static int feasibleBench(BMK_result_t* resultPtr, - const void* srcBuffer, const size_t srcSize, - void* dstBuffer, const size_t dstSize, - void* dictBuffer, const size_t dictSize, - const size_t* fileSizes, const size_t nbFiles, + const void* const * const srcPtrs, size_t const * const srcSizes, + void** const dstPtrs, size_t* dstCapacityToSizes, U32 const nbBlocks, + void* dictBuffer, const size_t dictBufferSize, ZSTD_CCtx* ctx, ZSTD_DCtx* dctx, const ZSTD_compressionParameters cParams, const constraint_t target, BMK_result_t* winnerResult) { - BMK_advancedParams_t adv = BMK_initAdvancedParams(); BMK_return_t benchres; U64 loopDurationC = 0, loopDurationD = 0; double uncertaintyConstantC, uncertaintyConstantD; - adv.loopMode = BMK_iterMode; - adv.nbSeconds = 1; //get ratio and 2x approx speed? - + size_t srcSize = 0; + U32 i; //alternative - test 1 iter for ratio, (possibility of error 3 which is fine), //maybe iter this until 2x measurable for better guarantee? DEBUGOUTPUT("Feas:\n"); - benchres = BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, fileSizes, nbFiles, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); + benchres = BMK_benchMemInvertible(srcPtrs, srcSizes, dstPtrs, dstCapacityToSizes, nbBlocks, 0, &cParams, dictBuffer, dictBufferSize, ctx, dctx, + BMK_both, BMK_iterMode, 1); if(benchres.error) { DISPLAY("ERROR %d !!\n", benchres.error); } + for(i = 0; i < nbBlocks; i++) { + srcSize += srcSizes[i]; + } BMK_printWinner(stdout, CUSTOM_LEVEL, benchres.result, cParams, srcSize); if(!benchres.error) { @@ -1151,9 +1491,8 @@ static int feasibleBench(BMK_result_t* resultPtr, if(benchres.result.cSize < winnerResult->cSize) { //better compression ratio, just needs to be feasible int feas; if(loopDurationC < TIMELOOP_NANOSEC / 10) { - BMK_return_t benchres2; - adv.mode = BMK_compressOnly; - benchres2 = BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, fileSizes, nbFiles, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); + BMK_return_t benchres2 = BMK_benchMemInvertible(srcPtrs, srcSizes, dstPtrs, dstCapacityToSizes, nbBlocks, 0, &cParams, dictBuffer, dictBufferSize, ctx, dctx, + BMK_compressOnly, BMK_iterMode, 1); if(benchres2.error) { return ERROR_RESULT; } else { @@ -1161,9 +1500,8 @@ static int feasibleBench(BMK_result_t* resultPtr, } } if(loopDurationD < TIMELOOP_NANOSEC / 10) { - BMK_return_t benchres2; - adv.mode = BMK_decodeOnly; - benchres2 = BMK_benchMemAdvanced(dstBuffer, dstSize, NULL, 0, &benchres.result.cSize, 1, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); + BMK_return_t benchres2 = BMK_benchMemInvertible((const void* const*)dstPtrs, dstCapacityToSizes, NULL, NULL, nbBlocks, 0, &cParams, dictBuffer, dictBufferSize, ctx, dctx, + BMK_decodeOnly, BMK_iterMode, 1); if(benchres2.error) { return ERROR_RESULT; } else { @@ -1174,11 +1512,9 @@ static int feasibleBench(BMK_result_t* resultPtr, feas = uncertainFeasibility(uncertaintyConstantC, uncertaintyConstantD, target, &(benchres.result)); if(feas == 0) { // uncertain feasibility - adv.loopMode = BMK_timeMode; if(loopDurationC < TIMELOOP_NANOSEC) { - BMK_return_t benchres2; - adv.mode = BMK_compressOnly; - benchres2 = BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, fileSizes, nbFiles, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); + BMK_return_t benchres2 = BMK_benchMemInvertible(srcPtrs, srcSizes, dstPtrs, dstCapacityToSizes, nbBlocks, 0, &cParams, dictBuffer, dictBufferSize, ctx, dctx, + BMK_compressOnly, BMK_timeMode, 1); if(benchres2.error) { return ERROR_RESULT; } else { @@ -1186,9 +1522,8 @@ static int feasibleBench(BMK_result_t* resultPtr, } } if(loopDurationD < TIMELOOP_NANOSEC) { - BMK_return_t benchres2; - adv.mode = BMK_decodeOnly; - benchres2 = BMK_benchMemAdvanced(dstBuffer,dstSize, NULL, 0, &benchres.result.cSize, 1, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); + BMK_return_t benchres2 = BMK_benchMemInvertible((const void* const*)dstPtrs, dstCapacityToSizes, NULL, NULL, nbBlocks, 0, &cParams, dictBuffer, dictBufferSize, ctx, dctx, + BMK_decodeOnly, BMK_timeMode, 1); if(benchres2.error) { return ERROR_RESULT; } else { @@ -1203,9 +1538,8 @@ static int feasibleBench(BMK_result_t* resultPtr, } else if (benchres.result.cSize == winnerResult->cSize) { //equal ratio, needs to be better than winner in cSpeed/ dSpeed / cMem int feas; if(loopDurationC < TIMELOOP_NANOSEC / 10) { - BMK_return_t benchres2; - adv.mode = BMK_compressOnly; - benchres2 = BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, fileSizes, nbFiles, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); + BMK_return_t benchres2 = BMK_benchMemInvertible(srcPtrs, srcSizes, dstPtrs, dstCapacityToSizes, nbBlocks, 0, &cParams, dictBuffer, dictBufferSize, ctx, dctx, + BMK_compressOnly, BMK_iterMode, 1); if(benchres2.error) { return ERROR_RESULT; } else { @@ -1213,9 +1547,8 @@ static int feasibleBench(BMK_result_t* resultPtr, } } if(loopDurationD < TIMELOOP_NANOSEC / 10) { - BMK_return_t benchres2; - adv.mode = BMK_decodeOnly; - benchres2 = BMK_benchMemAdvanced(dstBuffer, dstSize, NULL, 0, &benchres.result.cSize, 1, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); + BMK_return_t benchres2 = BMK_benchMemInvertible((const void* const*)dstPtrs, dstCapacityToSizes, NULL, NULL, nbBlocks, 0, &cParams, dictBuffer, dictBufferSize, ctx, dctx, + BMK_decodeOnly, BMK_iterMode, 1); if(benchres2.error) { return ERROR_RESULT; } else { @@ -1224,11 +1557,9 @@ static int feasibleBench(BMK_result_t* resultPtr, } feas = uncertainFeasibility(uncertaintyConstantC, uncertaintyConstantD, target, &(benchres.result)); if(feas == 0) { // uncertain feasibility - adv.loopMode = BMK_timeMode; if(loopDurationC < TIMELOOP_NANOSEC) { - BMK_return_t benchres2; - adv.mode = BMK_compressOnly; - benchres2 = BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, fileSizes, nbFiles, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); + BMK_return_t benchres2 = BMK_benchMemInvertible(srcPtrs, srcSizes, dstPtrs, dstCapacityToSizes, nbBlocks, 0, &cParams, dictBuffer, dictBufferSize, ctx, dctx, + BMK_compressOnly, BMK_timeMode, 1); if(benchres2.error) { return ERROR_RESULT; } else { @@ -1236,9 +1567,8 @@ static int feasibleBench(BMK_result_t* resultPtr, } } if(loopDurationD < TIMELOOP_NANOSEC) { - BMK_return_t benchres2; - adv.mode = BMK_decodeOnly; - benchres2 = BMK_benchMemAdvanced(dstBuffer,dstSize, NULL, 0, &benchres.result.cSize, 1, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); + BMK_return_t benchres2 = BMK_benchMemInvertible((const void* const*)dstPtrs, dstCapacityToSizes, NULL, NULL, nbBlocks, 0, &cParams, dictBuffer, dictBufferSize, ctx, dctx, + BMK_decodeOnly, BMK_timeMode, 1); if(benchres2.error) { return ERROR_RESULT; } else { @@ -1253,8 +1583,8 @@ static int feasibleBench(BMK_result_t* resultPtr, if(btw == -1) { return INFEASIBLE_RESULT; } else { //possibly better, benchmark and find out - adv.loopMode = BMK_timeMode; - benchres = BMK_benchMemAdvanced(srcBuffer, srcSize, dstBuffer, dstSize, fileSizes, nbFiles, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); + benchres = BMK_benchMemInvertible(srcPtrs, srcSizes, dstPtrs, dstCapacityToSizes, nbBlocks, 0, &cParams, dictBuffer, dictBufferSize, ctx, dctx, + BMK_both, BMK_timeMode, 1); *resultPtr = benchres.result; return objective_lt(*winnerResult, benchres.result); } @@ -1274,28 +1604,31 @@ static int feasibleBench(BMK_result_t* resultPtr, //have version of benchMemAdvanced which takes in dstBuffer/cap as well? //(motivation: repeat tests (maybe just on decompress) don't need further compress runs) static int infeasibleBench(BMK_result_t* resultPtr, - const void* srcBuffer, const size_t srcSize, - void* dstBuffer, const size_t dstSize, - void* dictBuffer, const size_t dictSize, - const size_t* fileSizes, const size_t nbFiles, - ZSTD_CCtx* ctx, ZSTD_DCtx* dctx, - const ZSTD_compressionParameters cParams, - const constraint_t target, - BMK_result_t* winnerResult) { - BMK_advancedParams_t adv = BMK_initAdvancedParams(); + const void* const * const srcPtrs, size_t const * const srcSizes, + void** const dstPtrs, size_t* dstCapacityToSizes, U32 const nbBlocks, + void* dictBuffer, const size_t dictBufferSize, + ZSTD_CCtx* ctx, ZSTD_DCtx* dctx, + const ZSTD_compressionParameters cParams, + const constraint_t target, + BMK_result_t* winnerResult) { BMK_return_t benchres; BMK_result_t resultMin, resultMax; U64 loopDurationC = 0, loopDurationD = 0; double uncertaintyConstantC, uncertaintyConstantD; - double winnerRS = resultScore(*winnerResult, srcSize, target); - adv.loopMode = BMK_iterMode; //can only use this for ratio measurement then, super inaccurate timing - adv.nbSeconds = 1; //get ratio and 2x approx speed? //maybe run until twice MIN(minloopinterval * clockDuration) + double winnerRS; + size_t srcSize = 0; + U32 i; + + benchres = BMK_benchMemInvertible(srcPtrs, srcSizes, dstPtrs, dstCapacityToSizes, nbBlocks, 0, &cParams, dictBuffer, dictBufferSize, ctx, dctx, + BMK_both, BMK_iterMode, 1); + for(i = 0; i < nbBlocks; i++) { + srcSize += srcSizes[i]; + } + BMK_printWinner(stdout, CUSTOM_LEVEL, benchres.result, cParams, srcSize); + winnerRS = resultScore(*winnerResult, srcSize, target); DEBUGOUTPUT("WinnerScore: %f\n ", winnerRS); - benchres = BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, fileSizes, nbFiles, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); - BMK_printWinner(stdout, CUSTOM_LEVEL, benchres.result, cParams, srcSize); - if(!benchres.error) { *resultPtr = benchres.result; if(eqZero(benchres.result.cSpeed)) { @@ -1316,9 +1649,8 @@ static int infeasibleBench(BMK_result_t* resultPtr, if(loopDurationC < TIMELOOP_NANOSEC / 10) { - BMK_return_t benchres2; - adv.mode = BMK_compressOnly; - benchres2 = BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, fileSizes, nbFiles, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); + BMK_return_t benchres2 = BMK_benchMemInvertible(srcPtrs, srcSizes, dstPtrs, dstCapacityToSizes, nbBlocks, 0, &cParams, dictBuffer, dictBufferSize, ctx, dctx, + BMK_compressOnly, BMK_iterMode, 1); if(benchres2.error) { return ERROR_RESULT; } else { @@ -1326,9 +1658,8 @@ static int infeasibleBench(BMK_result_t* resultPtr, } } if(loopDurationD < TIMELOOP_NANOSEC / 10) { - BMK_return_t benchres2; - adv.mode = BMK_decodeOnly; - benchres2 = BMK_benchMemAdvanced(dstBuffer, dstSize, NULL, 0, &benchres.result.cSize, 1, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); + BMK_return_t benchres2 = BMK_benchMemInvertible((const void* const*)dstPtrs, dstCapacityToSizes, NULL, NULL, nbBlocks, 0, &cParams, dictBuffer, dictBufferSize, ctx, dctx, + BMK_decodeOnly, BMK_iterMode, 1); if(benchres2.error) { return ERROR_RESULT; } else { @@ -1347,12 +1678,9 @@ static int infeasibleBench(BMK_result_t* resultPtr, if (winnerRS > resultScore(resultMax, srcSize, target)) { return INFEASIBLE_RESULT; } else { - //do this w/o copying / stuff - adv.loopMode = BMK_timeMode; if(loopDurationC < TIMELOOP_NANOSEC) { - BMK_return_t benchres2; - adv.mode = BMK_compressOnly; - benchres2 = BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, fileSizes, nbFiles, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); + BMK_return_t benchres2 = BMK_benchMemInvertible(srcPtrs, srcSizes, dstPtrs, dstCapacityToSizes, nbBlocks, 0, &cParams, dictBuffer, dictBufferSize, ctx, dctx, + BMK_compressOnly, BMK_timeMode, 1); if(benchres2.error) { return ERROR_RESULT; } else { @@ -1360,9 +1688,8 @@ static int infeasibleBench(BMK_result_t* resultPtr, } } if(loopDurationD < TIMELOOP_NANOSEC) { - BMK_return_t benchres2; - adv.mode = BMK_decodeOnly; - benchres2 = BMK_benchMemAdvanced(dstBuffer, dstSize, NULL, 0, &benchres.result.cSize, 1, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); + BMK_return_t benchres2 = BMK_benchMemInvertible((const void* const*)dstPtrs, dstCapacityToSizes, NULL, NULL, nbBlocks, 0, &cParams, dictBuffer, dictBufferSize, ctx, dctx, + BMK_decodeOnly, BMK_timeMode, 1); if(benchres2.error) { return ERROR_RESULT; } else { @@ -1394,16 +1721,62 @@ static int feasibleBenchMemo(BMK_result_t* resultPtr, const U32* varyParams, const int varyLen) { const size_t memind = memoTableInd(&cParams, varyParams, varyLen); - if(memoTable[memind] >= INFEASIBLE_THRESHOLD) { - return INFEASIBLE_RESULT; //probably pick a different code for already tested? - //maybe remove this if we incorporate nonrandom location picking? - //what is the intended behavior in this case? - //ignore? stop iterating completely? other? + return INFEASIBLE_RESULT; } else { - int res = feasibleBench(resultPtr, srcBuffer, srcSize, dstBuffer, dstSize, dictBuffer, dictSize, fileSizes, nbFiles, ctx, dctx, + const size_t blockSize = g_blockSize ? g_blockSize : srcSize; + U32 const maxNbBlocks = (U32) ((srcSize + (blockSize-1)) / blockSize) + nbFiles; + const void ** const srcPtrs = (const void** const)malloc(maxNbBlocks * sizeof(void*)); + size_t* const srcSizes = (size_t* const)malloc(maxNbBlocks * sizeof(size_t)); + void ** const dstPtrs = (void** const)malloc(maxNbBlocks * sizeof(void*)); + size_t* const dstCapacities = (size_t* const)malloc(maxNbBlocks * sizeof(size_t)); + U32 nbBlocks; + int res; + + if(!srcPtrs || !srcSizes || !dstPtrs || !dstCapacities) { + free(srcPtrs); + free(srcSizes); + free(dstPtrs); + free(dstCapacities); + DISPLAY("Allocation Error\n"); + return ERROR_RESULT; + } + + { + const char* srcPtr = (const char*)srcBuffer; + char* dstPtr = (char*)dstBuffer; + size_t dstSizeRemaining = dstSize; + U32 fileNb; + for (nbBlocks=0, fileNb=0; fileNb dstSizeRemaining) { + DEBUGOUTPUT("Warning: dstSize too small to benchmark completely \n"); + remaining = dstSizeRemaining; + dstSizeRemaining = 0; + } else { + dstSizeRemaining -= remaining; + } + for ( ; nbBlocks= INFEASIBLE_THRESHOLD) { return INFEASIBLE_RESULT; //see feasibleBenchMemo for concerns } else { - int res = infeasibleBench(resultPtr, srcBuffer, srcSize, dstBuffer, dstSize, dictBuffer, dictSize, fileSizes, nbFiles, ctx, dctx, + const size_t blockSize = g_blockSize ? g_blockSize : srcSize; + U32 const maxNbBlocks = (U32) ((srcSize + (blockSize-1)) / blockSize) + nbFiles; + const void ** const srcPtrs = (const void** const)malloc(maxNbBlocks * sizeof(void*)); + size_t* const srcSizes = (size_t* const)malloc(maxNbBlocks * sizeof(size_t)); + void ** const dstPtrs = (void** const)malloc(maxNbBlocks * sizeof(void*)); + size_t* const dstCapacities = (size_t* const)malloc(maxNbBlocks * sizeof(size_t)); + U32 nbBlocks; + int res; + + if(!srcPtrs || !srcSizes || !dstPtrs || !dstCapacities) { + free(srcPtrs); + free(srcSizes); + free(dstPtrs); + free(dstCapacities); + DISPLAY("Allocation Error\n"); + return ERROR_RESULT; + } + + { + const char* srcPtr = (const char*)srcBuffer; + char* dstPtr = (char*)dstBuffer; + size_t dstSizeRemaining = dstSize; + U32 fileNb; + for (nbBlocks=0, fileNb=0; fileNb dstSizeRemaining) { + DEBUGOUTPUT("Warning: dstSize too small to benchmark completely \n"); + remaining = dstSizeRemaining; + dstSizeRemaining = 0; + } else { + dstSizeRemaining -= remaining; + } + for ( ; nbBlocks candidate before restart } } + return winnerInfo; } @@ -1711,7 +2135,7 @@ static winnerInfo_t optimizeFixedStrategy( } while(i < 10) { //make i adjustable (user input?) depending on how much time they have. - DISPLAY("Restart\n"); //TODO: make better printing across restarts + DEBUGOUTPUT("Restart\n"); //look into improving this to maximize distance from searched infeasible stuff / towards promising regions? randomConstrainedParams(&init, varNew, varLenNew, memoTable); candidateInfo = climbOnce(target, varNew, varLenNew, memoTable, srcBuffer, srcSize, dstBuffer, dstSize, dictBuffer, dictSize, fileSizes, nbFiles, ctx, dctx, init); @@ -1918,7 +2342,6 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ /* find best solution from default params */ { /* strategy selection */ - //TODO: don't factor memory into strategy selection in constraint_t const int maxSeeds = g_noSeed ? 1 : ZSTD_maxCLevel(); DEBUGOUTPUT("Strategy Selection\n"); if(varLen == NUM_PARAMS && paramTarget.strategy == 0) { /* no variable based constraints */ @@ -2019,7 +2442,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ if(objective_lt(winner.result, wc.result)) { winner = wc; } - //TODO: we could double back to increase search of 'better' strategies + //We could double back to increase search of 'better' strategies st = nextStrategy(st, bestStrategy); } } else { From 0f91b039ff934c4fd42d292a90ea62b52cb2d696 Mon Sep 17 00:00:00 2001 From: George Lu Date: Mon, 16 Jul 2018 18:04:57 -0700 Subject: [PATCH 116/372] Add Levels --- tests/paramgrill.c | 39 ++++++++++++++++++++++++++++++++++----- 1 file changed, 34 insertions(+), 5 deletions(-) mode change 100644 => 100755 tests/paramgrill.c diff --git a/tests/paramgrill.c b/tests/paramgrill.c old mode 100644 new mode 100755 index e4d468846..086931b3d --- a/tests/paramgrill.c +++ b/tests/paramgrill.c @@ -56,7 +56,7 @@ static const int g_maxNbVariations = 64; #define DISPLAY(...) fprintf(stderr, __VA_ARGS__) #define TIMED 0 #ifndef DEBUG -# define DEBUG 0 +# define DEBUG 1 #endif #define DEBUGOUTPUT(...) { if (DEBUG) DISPLAY(__VA_ARGS__); } @@ -2227,7 +2227,7 @@ static int nextStrategy(const int currentStrategy, const int bestStrategy) { } //optimize fixed strategy. -static int optimizeForSize(const char* const * const fileNamesTable, const size_t nbFiles, const char* dictFileName, constraint_t target, ZSTD_compressionParameters paramTarget) +static int optimizeForSize(const char* const * const fileNamesTable, const size_t nbFiles, const char* dictFileName, constraint_t target, ZSTD_compressionParameters paramTarget, int cLevel) { size_t benchedSize; void* origBuff = NULL; @@ -2238,7 +2238,6 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ size_t* fileSizes = calloc(sizeof(size_t),nbFiles); const int varLen = variableParams(paramTarget, varArray); U8** allMT = NULL; - g_targetConstraints = target; g_winner.result.cSize = (size_t)-1; /* Init */ if(!cParamValid(paramTarget)) { @@ -2310,6 +2309,30 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ goto _cleanUp; } + //TODO: cLevel Stuff. + if(cLevel) { + BMK_result_t candidate; + const size_t blockSize = g_blockSize ? g_blockSize : benchedSize; + ZSTD_CCtx* const ctx = ZSTD_createCCtx(); + ZSTD_DCtx* const dctx = ZSTD_createDCtx(); + ZSTD_compressionParameters const CParams = ZSTD_getCParams(cLevel, blockSize, dictBufferSize); + if(BMK_benchParam(&candidate, origBuff, benchedSize, fileSizes, nbFiles, ctx, dctx, CParams)) { + ZSTD_freeCCtx(ctx); + ZSTD_freeDCtx(dctx); + ret = 3; + goto _cleanUp; + } + + target.cSpeed = candidate.cSpeed; //TODO: maybe have a small bit of slack here, like x.99? + target.dSpeed = candidate.dSpeed; + BMK_printWinner(stdout, cLevel, candidate, CParams, benchedSize); + + ZSTD_freeCCtx(ctx); + ZSTD_freeDCtx(dctx); + } + + g_targetConstraints = target; + /* bench */ DISPLAY("\r%79s\r", ""); if(nbFiles == 1) { @@ -2348,7 +2371,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ BMK_result_t candidate; int feas = 0, i; for (i=1; i<=maxSeeds; i++) { - ZSTD_compressionParameters const CParams = ZSTD_getCParams(i, blockSize, 0); + ZSTD_compressionParameters const CParams = ZSTD_getCParams(i, blockSize, dictBufferSize); int ec = BMK_benchParam(&candidate, origBuff, benchedSize, fileSizes, nbFiles, ctx, dctx, CParams); BMK_printWinner(stdout, i, candidate, CParams, benchedSize); @@ -2553,6 +2576,7 @@ int main(int argc, const char** argv) const char* dictFileName = 0; U32 optimizer = 0; U32 main_pause = 0; + int optimizerCLevel = 0; constraint_t target = { 0, 0, (U32)-1 }; //0 for anything unset @@ -2584,6 +2608,7 @@ int main(int argc, const char** argv) if (longCommandWArg(&argument, "compressionSpeed=") || longCommandWArg(&argument, "cSpeed=")) { target.cSpeed = readU32FromChar(&argument) * 1000000; if (argument[0]==',') { argument++; continue; } else break; } if (longCommandWArg(&argument, "decompressionSpeed=") || longCommandWArg(&argument, "dSpeed=")) { target.dSpeed = readU32FromChar(&argument) * 1000000; if (argument[0]==',') { argument++; continue; } else break; } if (longCommandWArg(&argument, "compressionMemory=") || longCommandWArg(&argument, "cMem=")) { target.cMem = readU32FromChar(&argument) * 1000000; if (argument[0]==',') { argument++; continue; } else break; } + //TODO: add Level; /* in MB or MB/s */ DISPLAY("invalid optimization parameter \n"); return 1; @@ -2692,6 +2717,10 @@ int main(int argc, const char** argv) argument++; paramTarget.strategy = (ZSTD_strategy)readU32FromChar(&argument); continue; + case 'L': /* level centers around a level */ + argument++; + optimizerCLevel = (int)readU32FromChar(&argument); + continue; default : ; } break; @@ -2796,7 +2825,7 @@ int main(int argc, const char** argv) } } else { if (optimizer) { - result = optimizeForSize(argv+filenamesStart, argc-filenamesStart, dictFileName, target, paramTarget); + result = optimizeForSize(argv+filenamesStart, argc-filenamesStart, dictFileName, target, paramTarget, optimizerCLevel); } else { result = benchFiles(argv+filenamesStart, argc-filenamesStart); } } From df026e159f7e5ff57a0caa5817176b00c059bc49 Mon Sep 17 00:00:00 2001 From: George Lu Date: Mon, 16 Jul 2018 18:22:04 -0700 Subject: [PATCH 117/372] Fix windows implicit casting bugs --- programs/bench.c | 4 ++-- tests/paramgrill.c | 40 ++++++++++++++++++++-------------------- 2 files changed, 22 insertions(+), 22 deletions(-) diff --git a/programs/bench.c b/programs/bench.c index 8e57db4c9..a34b74cf1 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -696,7 +696,7 @@ static BMK_return_t BMK_benchMemAdvancedNoAlloc( } DISPLAYLEVEL(2, "%2i#\n", cLevel); } /* Bench */ - results.result.cMem = (1 << (comprParams->windowLog)) + ZSTD_sizeof_CCtx(ctx); + results.result.cMem = (1ULL << (comprParams->windowLog)) + ZSTD_sizeof_CCtx(ctx); results.error = 0; return results; } @@ -731,7 +731,7 @@ BMK_return_t BMK_benchMemAdvanced(const void* srcBuffer, size_t srcSize, void* resultBuffer = malloc(srcSize); - BMK_return_t results; + BMK_return_t results = { { 0, 0, 0, 0 }, 0 }; int allocationincomplete; if(!dstCapacity) { diff --git a/tests/paramgrill.c b/tests/paramgrill.c index 086931b3d..02811b9c1 100755 --- a/tests/paramgrill.c +++ b/tests/paramgrill.c @@ -263,7 +263,9 @@ static int epsilonEqual(const double c1, const double c2) { /* checks exact equivalence to 0, to stop compiler complaining fpeq */ static int eqZero(const double c1) { - return (U64)c1 == (U64)0.0 || (U64)c1 == (U64)-0.0; + const double z1 = 0.0; + const double z2 = -0.0; + return !(memcmp(&c1, &z1, sizeof(double))) || !(memcmp(&c1, &z2, sizeof(double))); } /* returns 1 if result2 is strictly 'better' than result1 */ @@ -318,7 +320,7 @@ const char* g_stratName[ZSTD_btultra+1] = { "ZSTD_greedy ", "ZSTD_lazy ", "ZSTD_lazy2 ", "ZSTD_btlazy2 ", "ZSTD_btopt ", "ZSTD_btultra "}; -static size_t +static int BMK_benchParam(BMK_result_t* resultPtr, const void* srcBuffer, const size_t srcSize, const size_t* fileSizes, const unsigned nbFiles, @@ -331,7 +333,7 @@ BMK_benchParam(BMK_result_t* resultPtr, } /* benchParam but only takes in one file. */ -static size_t +static int BMK_benchParam1(BMK_result_t* resultPtr, const void* srcBuffer, size_t srcSize, ZSTD_CCtx* ctx, ZSTD_DCtx* dctx, @@ -1204,7 +1206,7 @@ static ZSTD_compressionParameters randomParams(void) /* Sets pc to random unmeasured set of parameters */ static void randomConstrainedParams(ZSTD_compressionParameters* pc, U32* varArray, int varLen, U8* memoTable) { - int tries = memoTableLen(varArray, varLen); //configurable, + size_t tries = memoTableLen(varArray, varLen); //configurable, const size_t maxSize = memoTableLen(varArray, varLen); size_t ind; do { @@ -1470,7 +1472,7 @@ static int feasibleBench(BMK_result_t* resultPtr, loopDurationC = 0; uncertaintyConstantC = 2; } else { - loopDurationC = ((srcSize * TIMELOOP_NANOSEC) / benchres.result.cSpeed); + loopDurationC = (U64)((double)(srcSize * TIMELOOP_NANOSEC) / benchres.result.cSpeed); //problem - tested in fullbench, saw speed vary 3x between iters, maybe raise uncertaintyConstraint up? //possibly has to do with initCCtx? or system stuff? //asymmetric +/- constant needed? @@ -1480,7 +1482,7 @@ static int feasibleBench(BMK_result_t* resultPtr, loopDurationD = 0; uncertaintyConstantD = 2; } else { - loopDurationD = ((srcSize * TIMELOOP_NANOSEC) / benchres.result.dSpeed); + loopDurationD = (U64)((double)(srcSize * TIMELOOP_NANOSEC) / benchres.result.dSpeed); //problem - tested in fullbench, saw speed vary 3x between iters, maybe raise uncertaintyConstraint up? //possibly has to do with initCCtx? or system stuff? //asymmetric +/- constant needed? @@ -1635,7 +1637,7 @@ static int infeasibleBench(BMK_result_t* resultPtr, loopDurationC = 0; uncertaintyConstantC = 2; } else { - loopDurationC = ((srcSize * TIMELOOP_NANOSEC) / benchres.result.cSpeed); + loopDurationC = (U64)((double)(srcSize * TIMELOOP_NANOSEC) / benchres.result.cSpeed); uncertaintyConstantC = MIN((loopDurationC + (double)(2 * g_clockGranularity)/loopDurationC * 1.1), 3); //.02 seconds } @@ -1643,7 +1645,7 @@ static int infeasibleBench(BMK_result_t* resultPtr, loopDurationD = 0; uncertaintyConstantD = 2; } else { - loopDurationD = ((srcSize * TIMELOOP_NANOSEC) / benchres.result.dSpeed); + loopDurationD = (U64)((double)(srcSize * TIMELOOP_NANOSEC) / benchres.result.dSpeed); uncertaintyConstantD = MIN((loopDurationD + (double)(2 * g_clockGranularity)/loopDurationD) * 1.1 , 3); //.02 seconds } @@ -1725,7 +1727,7 @@ static int feasibleBenchMemo(BMK_result_t* resultPtr, return INFEASIBLE_RESULT; } else { const size_t blockSize = g_blockSize ? g_blockSize : srcSize; - U32 const maxNbBlocks = (U32) ((srcSize + (blockSize-1)) / blockSize) + nbFiles; + U32 const maxNbBlocks = (U32) ((srcSize + (blockSize-1)) / blockSize) + (U32)nbFiles; const void ** const srcPtrs = (const void** const)malloc(maxNbBlocks * sizeof(void*)); size_t* const srcSizes = (size_t* const)malloc(maxNbBlocks * sizeof(size_t)); void ** const dstPtrs = (void** const)malloc(maxNbBlocks * sizeof(void*)); @@ -1798,7 +1800,7 @@ static int infeasibleBenchMemo(BMK_result_t* resultPtr, return INFEASIBLE_RESULT; //see feasibleBenchMemo for concerns } else { const size_t blockSize = g_blockSize ? g_blockSize : srcSize; - U32 const maxNbBlocks = (U32) ((srcSize + (blockSize-1)) / blockSize) + nbFiles; + U32 const maxNbBlocks = (U32) ((srcSize + (blockSize-1)) / blockSize) + (U32)nbFiles; const void ** const srcPtrs = (const void** const)malloc(maxNbBlocks * sizeof(void*)); size_t* const srcSizes = (size_t* const)malloc(maxNbBlocks * sizeof(size_t)); void ** const dstPtrs = (void** const)malloc(maxNbBlocks * sizeof(void*)); @@ -2279,7 +2281,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ } { - U64 const totalSizeToLoad = UTIL_getTotalFileSize(fileNamesTable, nbFiles); + U64 const totalSizeToLoad = UTIL_getTotalFileSize(fileNamesTable, (U32)nbFiles); int ec; unsigned i; benchedSize = BMK_findMaxMem(totalSizeToLoad * 3) / 3; @@ -2290,7 +2292,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ ret = 1; goto _cleanUp; } - ec = BMK_loadFiles(origBuff, benchedSize, fileSizes, fileNamesTable, nbFiles); + ec = BMK_loadFiles(origBuff, benchedSize, fileSizes, fileNamesTable, (U32)nbFiles); if(ec) { DISPLAY("Error Loading Files"); ret = ec; @@ -2308,23 +2310,21 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ ret = 2; goto _cleanUp; } - - //TODO: cLevel Stuff. + if(cLevel) { BMK_result_t candidate; const size_t blockSize = g_blockSize ? g_blockSize : benchedSize; ZSTD_CCtx* const ctx = ZSTD_createCCtx(); ZSTD_DCtx* const dctx = ZSTD_createDCtx(); ZSTD_compressionParameters const CParams = ZSTD_getCParams(cLevel, blockSize, dictBufferSize); - if(BMK_benchParam(&candidate, origBuff, benchedSize, fileSizes, nbFiles, ctx, dctx, CParams)) { + if(BMK_benchParam(&candidate, origBuff, benchedSize, fileSizes, (U32)nbFiles, ctx, dctx, CParams)) { ZSTD_freeCCtx(ctx); ZSTD_freeDCtx(dctx); ret = 3; goto _cleanUp; } - target.cSpeed = candidate.cSpeed; //TODO: maybe have a small bit of slack here, like x.99? - target.dSpeed = candidate.dSpeed; + target.cSpeed = (U32)candidate.cSpeed; //Maybe have a small bit of slack here, like x.99? BMK_printWinner(stdout, cLevel, candidate, CParams, benchedSize); ZSTD_freeCCtx(ctx); @@ -2372,7 +2372,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ int feas = 0, i; for (i=1; i<=maxSeeds; i++) { ZSTD_compressionParameters const CParams = ZSTD_getCParams(i, blockSize, dictBufferSize); - int ec = BMK_benchParam(&candidate, origBuff, benchedSize, fileSizes, nbFiles, ctx, dctx, CParams); + int ec = BMK_benchParam(&candidate, origBuff, benchedSize, fileSizes, (U32)nbFiles, ctx, dctx, CParams); BMK_printWinner(stdout, i, candidate, CParams, benchedSize); if(!ec) { @@ -2408,7 +2408,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ randomConstrainedParams(&candidateParams, varNew, varLenNew, allMT[i]); cParamZeroMin(&candidateParams); candidateParams = sanitizeParams(candidateParams); - ec = BMK_benchParam(&candidate, origBuff, benchedSize, fileSizes, nbFiles, ctx, dctx, candidateParams); + ec = BMK_benchParam(&candidate, origBuff, benchedSize, fileSizes, (U32)nbFiles, ctx, dctx, candidateParams); if(!ec) { if(feas) { @@ -2608,7 +2608,7 @@ int main(int argc, const char** argv) if (longCommandWArg(&argument, "compressionSpeed=") || longCommandWArg(&argument, "cSpeed=")) { target.cSpeed = readU32FromChar(&argument) * 1000000; if (argument[0]==',') { argument++; continue; } else break; } if (longCommandWArg(&argument, "decompressionSpeed=") || longCommandWArg(&argument, "dSpeed=")) { target.dSpeed = readU32FromChar(&argument) * 1000000; if (argument[0]==',') { argument++; continue; } else break; } if (longCommandWArg(&argument, "compressionMemory=") || longCommandWArg(&argument, "cMem=")) { target.cMem = readU32FromChar(&argument) * 1000000; if (argument[0]==',') { argument++; continue; } else break; } - //TODO: add Level; + if (longCommandWArg(&argument, "level=") || longCommandWArg(&argument, "lvl=")) { optimizerCLevel = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; } /* in MB or MB/s */ DISPLAY("invalid optimization parameter \n"); return 1; From e148db366ea424bbc2dc7269c566adb97cc11ec6 Mon Sep 17 00:00:00 2001 From: George Lu Date: Fri, 20 Jul 2018 14:35:09 -0700 Subject: [PATCH 118/372] Separate capacity vs size Also: Make suggested fixes -varInds_t -reorder some arguments -remove code duplication -update README / -h -Fix memory leaks --- programs/bench.c | 63 +- programs/bench.h | 8 +- tests/README.md | 32 + tests/fullbench.c | 2 +- tests/paramgrill.c | 1963 ++++++++++++++++---------------------------- 5 files changed, 779 insertions(+), 1289 deletions(-) mode change 100755 => 100644 tests/paramgrill.c diff --git a/programs/bench.c b/programs/bench.c index a34b74cf1..177dbe0e3 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -171,6 +171,8 @@ struct BMK_timeState_t{ static void BMK_initCCtx(ZSTD_CCtx* ctx, const void* dictBuffer, size_t dictBufferSize, int cLevel, const ZSTD_compressionParameters* comprParams, const BMK_advancedParams_t* adv) { + ZSTD_CCtx_reset(ctx); + ZSTD_CCtx_resetParameters(ctx); if (adv->nbWorkers==1) { ZSTD_CCtx_setParameter(ctx, ZSTD_p_nbWorkers, 0); } else { @@ -195,6 +197,7 @@ static void BMK_initCCtx(ZSTD_CCtx* ctx, static void BMK_initDCtx(ZSTD_DCtx* dctx, const void* dictBuffer, size_t dictBufferSize) { + ZSTD_DCtx_reset(dctx); ZSTD_DCtx_loadDictionary(dctx, dictBuffer, dictBufferSize); } @@ -291,9 +294,9 @@ BMK_customReturn_t BMK_benchFunction( BMK_initFn_t initFn, void* initPayload, size_t blockCount, const void* const * const srcBlockBuffers, const size_t* srcBlockSizes, - void* const * const dstBlockBuffers, size_t* dstBlockCapacitiesToSizes, + void* const * const dstBlockBuffers, const size_t* dstBlockCapacities, size_t* cSizes, unsigned nbLoops) { - size_t srcSize = 0, dstSize = 0, ind = 0; + size_t dstSize = 0; U64 totalTime; BMK_customReturn_t retval; @@ -303,36 +306,37 @@ BMK_customReturn_t BMK_benchFunction( EXM_THROW_ND(1, BMK_customReturn_t, "nbLoops must be nonzero \n"); } - for(ind = 0; ind < blockCount; ind++) { - srcSize += srcBlockSizes[ind]; - } - { size_t i; for(i = 0; i < blockCount; i++) { - memset(dstBlockBuffers[i], 0xE5, dstBlockCapacitiesToSizes[i]); /* warm up and erase result buffer */ + memset(dstBlockBuffers[i], 0xE5, dstBlockCapacities[i]); /* warm up and erase result buffer */ } - - //UTIL_sleepMilli(5); /* give processor time to other processes */ - //UTIL_waitForNextTick(); +#if 0 + /* based on testing these seem to lower accuracy of multiple calls of 1 nbLoops vs 1 call of multiple nbLoops + * (Makes former slower) + */ + UTIL_sleepMilli(5); /* give processor time to other processes */ + UTIL_waitForNextTick(); +#endif } { - unsigned i, j, firstIter = 1; + unsigned i, j; clockStart = UTIL_getTime(); if(initFn != NULL) { initFn(initPayload); } for(i = 0; i < nbLoops; i++) { for(j = 0; j < blockCount; j++) { - size_t res = benchFn(srcBlockBuffers[j], srcBlockSizes[j], dstBlockBuffers[j], dstBlockCapacitiesToSizes[j], benchPayload); + size_t res = benchFn(srcBlockBuffers[j], srcBlockSizes[j], dstBlockBuffers[j], dstBlockCapacities[j], benchPayload); if(ZSTD_isError(res)) { EXM_THROW_ND(2, BMK_customReturn_t, "Function benchmarking failed on block %u of size %u : %s \n", - j, (U32)dstBlockCapacitiesToSizes[j], ZSTD_getErrorName(res)); - } else if(firstIter) { + j, (U32)dstBlockCapacities[j], ZSTD_getErrorName(res)); + } else if(i == nbLoops - 1) { dstSize += res; - dstBlockCapacitiesToSizes[j] = res; + if(cSizes != NULL) { + cSizes[j] = res; + } } } - firstIter = 0; } totalTime = UTIL_clockSpanNano(clockStart); } @@ -369,7 +373,7 @@ BMK_customTimedReturn_t BMK_benchFunctionTimed( BMK_initFn_t initFn, void* initPayload, size_t blockCount, const void* const* const srcBlockBuffers, const size_t* srcBlockSizes, - void * const * const dstBlockBuffers, size_t * dstBlockCapacitiesToSizes) + void * const * const dstBlockBuffers, const size_t * dstBlockCapacities, size_t* dstSizes) { U64 fastest = cont->fastestTime; int completed = 0; @@ -384,9 +388,9 @@ BMK_customTimedReturn_t BMK_benchFunctionTimed( UTIL_sleep(COOLPERIOD_SEC); cont->coolTime = UTIL_getTime(); } - + /* reinitialize capacity */ r.result = BMK_benchFunction(benchFn, benchPayload, initFn, initPayload, - blockCount, srcBlockBuffers, srcBlockSizes, dstBlockBuffers, dstBlockCapacitiesToSizes, cont->nbLoops); + blockCount, srcBlockBuffers, srcBlockSizes, dstBlockBuffers, dstBlockCapacities, dstSizes, cont->nbLoops); if(r.result.error) { /* completed w/ error */ r.completed = 1; return r; @@ -420,7 +424,7 @@ BMK_customTimedReturn_t BMK_benchFunctionTimed( /* benchMem with no allocation */ static BMK_return_t BMK_benchMemAdvancedNoAlloc( const void ** const srcPtrs, size_t* const srcSizes, - void** const cPtrs, size_t* const cSizes, + void** const cPtrs, size_t* const cCapacities, size_t* const cSizes, void** const resPtrs, size_t* const resSizes, void** resultBufferPtr, void* compressedBuffer, const size_t maxCompressedSize, @@ -485,11 +489,11 @@ static BMK_return_t BMK_benchMemAdvancedNoAlloc( srcPtrs[nbBlocks] = (const void*)srcPtr; srcSizes[nbBlocks] = thisBlockSize; cPtrs[nbBlocks] = (void*)cPtr; - cSizes[nbBlocks] = (adv->mode == BMK_decodeOnly) ? thisBlockSize : ZSTD_compressBound(thisBlockSize); + cCapacities[nbBlocks] = (adv->mode == BMK_decodeOnly) ? thisBlockSize : ZSTD_compressBound(thisBlockSize); resPtrs[nbBlocks] = (void*)resPtr; resSizes[nbBlocks] = (adv->mode == BMK_decodeOnly) ? (size_t) ZSTD_findDecompressedSize(srcPtr, thisBlockSize) : thisBlockSize; srcPtr += thisBlockSize; - cPtr += cSizes[nbBlocks]; + cPtr += cCapacities[nbBlocks]; resPtr += thisBlockSize; remaining -= thisBlockSize; } @@ -540,7 +544,7 @@ static BMK_return_t BMK_benchMemAdvancedNoAlloc( while(!(intermediateResultCompress.completed && intermediateResultDecompress.completed)) { if(!intermediateResultCompress.completed) { intermediateResultCompress = BMK_benchFunctionTimed(timeStateCompress, &local_defaultCompress, (void*)ctx, &local_initCCtx, (void*)&cctxprep, - nbBlocks, srcPtrs, srcSizes, cPtrs, cSizes); + nbBlocks, srcPtrs, srcSizes, cPtrs, cCapacities, cSizes); if(intermediateResultCompress.result.error) { results.error = intermediateResultCompress.result.error; return results; @@ -564,7 +568,7 @@ static BMK_return_t BMK_benchMemAdvancedNoAlloc( if(!intermediateResultDecompress.completed) { intermediateResultDecompress = BMK_benchFunctionTimed(timeStateDecompress, &local_defaultDecompress, (void*)(dctx), &local_initDCtx, (void*)&dctxprep, - nbBlocks, (const void* const*)cPtrs, cSizes, resPtrs, resSizes); + nbBlocks, (const void* const*)cPtrs, cSizes, resPtrs, resSizes, NULL); if(intermediateResultDecompress.result.error) { results.error = intermediateResultDecompress.result.error; return results; @@ -590,7 +594,7 @@ static BMK_return_t BMK_benchMemAdvancedNoAlloc( if(adv->mode != BMK_decodeOnly) { BMK_customReturn_t compressionResults = BMK_benchFunction(&local_defaultCompress, (void*)ctx, &local_initCCtx, (void*)&cctxprep, - nbBlocks, srcPtrs, srcSizes, cPtrs, cSizes, adv->nbSeconds); + nbBlocks, srcPtrs, srcSizes, cPtrs, cCapacities, cSizes, adv->nbSeconds); if(compressionResults.error) { results.error = compressionResults.error; return results; @@ -617,7 +621,7 @@ static BMK_return_t BMK_benchMemAdvancedNoAlloc( BMK_customReturn_t decompressionResults = BMK_benchFunction( &local_defaultDecompress, (void*)(dctx), &local_initDCtx, (void*)&dctxprep, nbBlocks, - (const void* const*)cPtrs, cSizes, resPtrs, resSizes, + (const void* const*)cPtrs, cSizes, resPtrs, resSizes, NULL, adv->nbSeconds); if(decompressionResults.error) { results.error = decompressionResults.error; @@ -717,8 +721,10 @@ BMK_return_t BMK_benchMemAdvanced(const void* srcBuffer, size_t srcSize, const void ** const srcPtrs = (const void** const)malloc(maxNbBlocks * sizeof(void*)); size_t* const srcSizes = (size_t* const)malloc(maxNbBlocks * sizeof(size_t)); + void ** const cPtrs = (void** const)malloc(maxNbBlocks * sizeof(void*)); size_t* const cSizes = (size_t* const)malloc(maxNbBlocks * sizeof(size_t)); + size_t* const cCapacities = (size_t* const)malloc(maxNbBlocks * sizeof(size_t)); void ** const resPtrs = (void** const)malloc(maxNbBlocks * sizeof(void*)); size_t* const resSizes = (size_t* const)malloc(maxNbBlocks * sizeof(size_t)); @@ -744,13 +750,11 @@ BMK_return_t BMK_benchMemAdvanced(const void* srcBuffer, size_t srcSize, !srcPtrs || !srcSizes || !cPtrs || !cSizes || !resPtrs || !resSizes; if (!allocationincomplete) { - results = BMK_benchMemAdvancedNoAlloc(srcPtrs, srcSizes, cPtrs, cSizes, + results = BMK_benchMemAdvancedNoAlloc(srcPtrs, srcSizes, cPtrs, cCapacities, cSizes, resPtrs, resSizes, &resultBuffer, compressedBuffer, maxCompressedSize, timeStateCompress, timeStateDecompress, srcBuffer, srcSize, fileSizes, nbFiles, cLevel, comprParams, dictBuffer, dictBufferSize, ctx, dctx, displayLevel, displayName, adv); } - - /* clean up */ BMK_freeTimeState(timeStateCompress); @@ -764,6 +768,7 @@ BMK_return_t BMK_benchMemAdvanced(const void* srcBuffer, size_t srcSize, free(srcSizes); free(cPtrs); free(cSizes); + free(cCapacities); free(resPtrs); free(resSizes); diff --git a/programs/bench.h b/programs/bench.h index 25d9f24a7..2a9945ac8 100644 --- a/programs/bench.h +++ b/programs/bench.h @@ -177,6 +177,7 @@ typedef size_t (*BMK_initFn_t)(void*); * dstBuffers - an array of buffers to be written into by benchFn * dstCapacitiesToSizes - an array of the capacities of above buffers. Output modified to compressed sizes of those blocks. * nbLoops - defines number of times benchFn is run. + * assumed array of size blockCount, will have compressed size of each block written to it. * return * .error will give a nonzero value if ZSTD_isError() is nonzero for any of the return * of the calls to initFn and benchFn, or if benchFunction errors internally @@ -187,12 +188,11 @@ typedef size_t (*BMK_initFn_t)(void*); * into dstBuffer, hence this value will be the total amount of bytes written to * dstBuffer. */ -BMK_customReturn_t BMK_benchFunction( - BMK_benchFn_t benchFn, void* benchPayload, +BMK_customReturn_t BMK_benchFunction(BMK_benchFn_t benchFn, void* benchPayload, BMK_initFn_t initFn, void* initPayload, size_t blockCount, const void* const * const srcBuffers, const size_t* srcSizes, - void * const * const dstBuffers, size_t* dstCapacitiesToSizes, + void * const * const dstBuffers, const size_t* dstCapacities, size_t* cSizes, unsigned nbLoops); @@ -221,7 +221,7 @@ BMK_customTimedReturn_t BMK_benchFunctionTimed(BMK_timedFnState_t* cont, BMK_initFn_t initFn, void* initPayload, size_t blockCount, const void* const * const srcBlockBuffers, const size_t* srcBlockSizes, - void* const * const dstBlockBuffers, size_t* dstBlockCapacitiesToSizes); + void* const * const dstBlockBuffers, const size_t* dstBlockCapacities, size_t* cSizes); #endif /* BENCH_H_121279284357 */ diff --git a/tests/README.md b/tests/README.md index 24a28ab7b..8bedd0a3c 100644 --- a/tests/README.md +++ b/tests/README.md @@ -88,3 +88,35 @@ as well as the 10,000 original files for more detailed comparison of decompressi will choose a random seed, and for 1 minute, generate random test frames and ensure that the zstd library correctly decompresses them in both simple and streaming modes. + +#### `paramgrill` - tool for generating compression table parameters and optimizing parameters on file given constraints + +Full list of arguments +``` + -T# : set level 1 speed objective + -B# : cut input into blocks of size # (default : single block) + -i# : iteration loops + -S : benchmarks a single run (example command: -Sl3w10h12) + w# - windowLog + h# - hashLog + c# - chainLog + s# - searchLog + l# - searchLength + t# - targetLength + S# - strategy + L# - level + --zstd= : Single run, parameter selection syntax same as zstdcli + --optimize= : find parameters to maximize compression ratio given parameters + Can use all --zstd= commands to constrain the type of solution found in addition to the following constraints + cSpeed= - Minimum compression speed + dSpeed= - Minimum decompression speed + cMem= - compression memory + lvl= - Automatically sets compression speed constraint to the speed of that level + --optimize= : same as -O with more verbose syntax + -P# : generated sample compressibility + -t# : Caps runtime of operation in seconds (default : 99999 seconds (about 27 hours )) + -v : Prints Benchmarking output + -D : Next argument dictionary file + +``` + Any inputs afterwards are treated as files to benchmark. diff --git a/tests/fullbench.c b/tests/fullbench.c index 9e7639f92..270cac86a 100644 --- a/tests/fullbench.c +++ b/tests/fullbench.c @@ -516,7 +516,7 @@ static size_t benchMem(const void* src, size_t srcSize, U32 benchNb, int cLevel, { r = BMK_benchFunction(benchFunction, buff2, NULL, NULL, 1, &src, &srcSize, - (void **)&dstBuff, &dstBuffSize, g_nbIterations); + (void **)&dstBuff, &dstBuffSize, NULL, g_nbIterations); if(r.error) { DISPLAY("ERROR %d ! ! \n", r.error); errorcode = r.error; diff --git a/tests/paramgrill.c b/tests/paramgrill.c old mode 100755 new mode 100644 index 02811b9c1..099be3688 --- a/tests/paramgrill.c +++ b/tests/paramgrill.c @@ -56,7 +56,7 @@ static const int g_maxNbVariations = 64; #define DISPLAY(...) fprintf(stderr, __VA_ARGS__) #define TIMED 0 #ifndef DEBUG -# define DEBUG 1 +# define DEBUG 0 #endif #define DEBUGOUTPUT(...) { if (DEBUG) DISPLAY(__VA_ARGS__); } @@ -67,14 +67,15 @@ static const int g_maxNbVariations = 64; #define CUSTOM_LEVEL 99 /* indices for each of the variables */ -#define WLOG_IND 0 -#define CLOG_IND 1 -#define HLOG_IND 2 -#define SLOG_IND 3 -#define SLEN_IND 4 -#define TLEN_IND 5 -//#define STRT_IND 6 -//#define NUM_PARAMS 7 +typedef enum { + wlog_ind = 0, + clog_ind = 1, + hlog_ind = 2, + slog_ind = 3, + slen_ind = 4, + tlen_ind = 5 +} varInds_t; + #define NUM_PARAMS 6 //just don't use strategy as a param. @@ -85,20 +86,16 @@ static const int g_maxNbVariations = 64; #define ZSTD_TARGETLENGTH_MIN 0 #define ZSTD_TARGETLENGTH_MAX 999 -//#define ZSTD_TARGETLENGTH_MAX 1024 #define WLOG_RANGE (ZSTD_WINDOWLOG_MAX - ZSTD_WINDOWLOG_MIN + 1) #define CLOG_RANGE (ZSTD_CHAINLOG_MAX - ZSTD_CHAINLOG_MIN + 1) #define HLOG_RANGE (ZSTD_HASHLOG_MAX - ZSTD_HASHLOG_MIN + 1) #define SLOG_RANGE (ZSTD_SEARCHLOG_MAX - ZSTD_SEARCHLOG_MIN + 1) #define SLEN_RANGE (ZSTD_SEARCHLENGTH_MAX - ZSTD_SEARCHLENGTH_MIN + 1) -#define TLEN_RANGE 12 -//TLEN_RANGE = 0, 2^0 to 2^10; -//hard coded since we only use powers of 2 (and 999 ~ 1024) +#define TLEN_RANGE 17 +/* TLEN_RANGE picked manually */ -//static const int mintable[NUM_PARAMS] = { ZSTD_WINDOWLOG_MIN, ZSTD_CHAINLOG_MIN, ZSTD_HASHLOG_MIN, ZSTD_SEARCHLOG_MIN, ZSTD_SEARCHLENGTH_MIN, ZSTD_TARGETLENGTH_MIN }; -//static const int maxtable[NUM_PARAMS] = { ZSTD_WINDOWLOG_MAX, ZSTD_CHAINLOG_MAX, ZSTD_HASHLOG_MAX, ZSTD_SEARCHLOG_MAX, ZSTD_SEARCHLENGTH_MAX, ZSTD_TARGETLENGTH_MAX }; static const int rangetable[NUM_PARAMS] = { WLOG_RANGE, CLOG_RANGE, HLOG_RANGE, SLOG_RANGE, SLEN_RANGE, TLEN_RANGE }; - +static const U32 tlen_table[TLEN_RANGE] = { 0, 1, 2, 4, 6, 8, 12, 16, 24, 32, 48, 64, 96, 128, 256, 512, 999 }; /*-************************************ * Benchmark Parameters **************************************/ @@ -179,9 +176,6 @@ static unsigned longCommandWArg(const char** stringPtr, const char* longCommand) return result; } -//assume that clock can at least measure .01 second intervals? -//make this a settable global initialized with fn? -//#define CLOCK_GRANULARITY 100000000ULL static U64 g_clockGranularity = 100000000ULL; static void findClockGranularity(void) { @@ -253,7 +247,7 @@ static void BMK_translateAdvancedParams(const ZSTD_compressionParameters params) /* checks results are feasible */ static int feasible(const BMK_result_t results, const constraint_t target) { - return (results.cSpeed >= target.cSpeed) && (results.dSpeed >= target.dSpeed) && (results.cMem <= target.cMem || !target.cMem); + return (results.cSpeed >= target.cSpeed) && (results.dSpeed >= target.dSpeed) && (results.cMem <= target.cMem); } #define EPSILON 0.001 @@ -268,13 +262,6 @@ static int eqZero(const double c1) { return !(memcmp(&c1, &z1, sizeof(double))) || !(memcmp(&c1, &z2, sizeof(double))); } -/* returns 1 if result2 is strictly 'better' than result1 */ -/* strict comparison / cutoff based */ -static int objective_lt(const BMK_result_t result1, const BMK_result_t result2) { - return (result1.cSize > result2.cSize) || (result1.cSize == result2.cSize && result2.cSpeed > result1.cSpeed) - || (result1.cSize == result2.cSize && epsilonEqual(result2.cSpeed, result1.cSpeed) && result2.dSpeed > result1.dSpeed); -} - /* hill climbing value for part 1 */ static double resultScore(const BMK_result_t res, const size_t srcSize, const constraint_t target) { double cs = 0., ds = 0., rt, cm = 0.; @@ -291,6 +278,16 @@ static double resultScore(const BMK_result_t res, const size_t srcSize, const co return ret; } +/* return true if r2 strictly better than r1 */ +static int compareResultLT(const BMK_result_t result1, const BMK_result_t result2, const constraint_t target, size_t srcSize) { + if(feasible(result1, target) && feasible(result2, target)) { + return (result1.cSize > result2.cSize) || (result1.cSize == result2.cSize && result2.cSpeed > result1.cSpeed) + || (result1.cSize == result2.cSize && epsilonEqual(result2.cSpeed, result1.cSpeed) && result2.dSpeed > result1.dSpeed); + } + return feasible(result2, target) || (!feasible(result1, target) && (resultScore(result1, srcSize, target) < resultScore(result2, srcSize, target))); + +} + /* factor sort of arbitrary */ static constraint_t relaxTarget(constraint_t target) { target.cMem = (U32)-1; @@ -303,35 +300,11 @@ static constraint_t relaxTarget(constraint_t target) { * Bench functions *********************************************************/ -typedef struct -{ - const char* srcPtr; - size_t srcSize; - char* cPtr; - size_t cRoom; - size_t cSize; - char* resPtr; - size_t resSize; -} blockParam_t; - - const char* g_stratName[ZSTD_btultra+1] = { "(none) ", "ZSTD_fast ", "ZSTD_dfast ", "ZSTD_greedy ", "ZSTD_lazy ", "ZSTD_lazy2 ", "ZSTD_btlazy2 ", "ZSTD_btopt ", "ZSTD_btultra "}; -static int -BMK_benchParam(BMK_result_t* resultPtr, - const void* srcBuffer, const size_t srcSize, - const size_t* fileSizes, const unsigned nbFiles, - ZSTD_CCtx* ctx, ZSTD_DCtx* dctx, - const ZSTD_compressionParameters cParams) { - - BMK_return_t res = BMK_benchMem(srcBuffer,srcSize, fileSizes, nbFiles, 0, &cParams, NULL, 0, ctx, dctx, 0, "File"); - *resultPtr = res.result; - return res.error; -} - /* benchParam but only takes in one file. */ static int BMK_benchParam1(BMK_result_t* resultPtr, @@ -349,6 +322,49 @@ typedef struct { ZSTD_compressionParameters params; } winnerInfo_t; +static ZSTD_compressionParameters emptyParams(void) { + ZSTD_compressionParameters p = { 0, 0, 0, 0, 0, 0, (ZSTD_strategy)0 }; + return p; +} + +static winnerInfo_t initWinnerInfo(ZSTD_compressionParameters p) { + winnerInfo_t w1; + w1.result.cSpeed = 0.; + w1.result.dSpeed = 0.; + w1.result.cMem = (size_t)-1; + w1.result.cSize = (size_t)-1; + w1.params = p; + return w1; +} + +typedef struct { + size_t srcSize; + void** srcPtrs; + size_t* srcSizes; + void** dstPtrs; + size_t* dstCapacities; + size_t* dstSizes; + void** resPtrs; + size_t* resSizes; + size_t nbBlocks; +} buffers_t; + +typedef struct { + size_t dictSize; + void* dictBuffer; + ZSTD_CCtx* cctx; + ZSTD_DCtx* dctx; +} contexts_t; + +static int +BMK_benchParam(BMK_result_t* resultPtr, + buffers_t buf, contexts_t ctx, + const ZSTD_compressionParameters cParams) { + BMK_return_t res = BMK_benchMem(buf.srcPtrs[0], buf.srcSize, buf.srcSizes, (unsigned)buf.nbBlocks, 0, &cParams, ctx.dictBuffer, ctx.dictSize, ctx.cctx, ctx.dctx, 0, "Files"); + *resultPtr = res.result; + return res.error; +} + /*-******************************************************* * From Paramgrill *********************************************************/ @@ -356,6 +372,8 @@ typedef struct { static void BMK_initCCtx(ZSTD_CCtx* ctx, const void* dictBuffer, size_t dictBufferSize, int cLevel, const ZSTD_compressionParameters* comprParams, const BMK_advancedParams_t* adv) { + ZSTD_CCtx_reset(ctx); + ZSTD_CCtx_resetParameters(ctx); if (adv->nbWorkers==1) { ZSTD_CCtx_setParameter(ctx, ZSTD_p_nbWorkers, 0); } else { @@ -380,6 +398,7 @@ static void BMK_initCCtx(ZSTD_CCtx* ctx, static void BMK_initDCtx(ZSTD_DCtx* dctx, const void* dictBuffer, size_t dictBufferSize) { + ZSTD_DCtx_reset(dctx); ZSTD_DCtx_loadDictionary(dctx, dictBuffer, dictBufferSize); } @@ -425,6 +444,7 @@ static size_t local_defaultCompress( out.dst = dstBuffer; out.size = dstSize; out.pos = 0; + assert(dstSize == ZSTD_compressBound(srcSize)); /* specific to this version, which is only used in paramgrill */ while (moreToFlush) { if(out.pos == out.size) { return (size_t)-ZSTD_error_dstSize_tooSmall; @@ -471,249 +491,164 @@ static size_t local_defaultDecompress( *********************************************************/ /* Replicate function of benchMemAdvanced, but with pre-split src / dst buffers, with relevant info to invert it (compressedSizes) passed out. */ -/*BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, fileSizes, nbFiles, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); */ +/* BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, fileSizes, nbFiles, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); */ /* nbSeconds used in same way as in BMK_advancedParams_t, as nbIters when in iterMode */ /* if in decodeOnly, then srcPtr's will be compressed blocks, and uncompressedBlocks will be written to dstPtrs? */ /* dictionary nullable, nothing else though. */ -static BMK_return_t BMK_benchMemInvertible(const void * const * const srcPtrs, size_t const * const srcSizes, - void** dstPtrs, size_t* dstCapacityToSizes, U32 const nbBlocks, +static BMK_return_t BMK_benchMemInvertible(buffers_t buf, contexts_t ctx, const int cLevel, const ZSTD_compressionParameters* comprParams, - const void* dictBuffer, const size_t dictBufferSize, - ZSTD_CCtx* ctx, ZSTD_DCtx* dctx, const BMK_mode_t mode, const BMK_loopMode_t loopMode, const unsigned nbSeconds) { + U32 i; BMK_return_t results = { { 0, 0., 0., 0 }, 0 } ; - size_t srcSize = 0; - void** const resPtrs = malloc(sizeof(void*) * nbBlocks); /* only really needed in both mode. */ - size_t* const resSizes = malloc(sizeof(size_t) * nbBlocks); - int freeDST = 0; + const void *const *const srcPtrs = (const void *const *const)buf.srcPtrs; + size_t const *const srcSizes = buf.srcSizes; + void** dstPtrs = buf.dstPtrs; + size_t* dstCapacities = buf.dstCapacities; + size_t* dstSizes = buf.dstSizes; + void** resPtrs = buf.resPtrs; + size_t* resSizes = buf.resSizes; + const void* dictBuffer = ctx.dictBuffer; + const size_t dictBufferSize = ctx.dictSize; + const size_t nbBlocks = buf.nbBlocks; + const size_t srcSize = buf.srcSize; + ZSTD_CCtx* cctx = ctx.cctx; + ZSTD_DCtx* dctx = ctx.dctx; BMK_advancedParams_t adv = BMK_initAdvancedParams(); adv.mode = mode; adv.loopMode = loopMode; adv.nbSeconds = nbSeconds; - /* resSizes == srcSizes, but modifiable */ - memcpy(resSizes, srcSizes, sizeof(size_t) * nbBlocks); - - for(i = 0; i < nbBlocks; i++) { - srcSize += srcSizes[i]; - } - - if(!ctx || !dctx || !srcPtrs || ! srcSizes) - { - results.error = 31; - DISPLAY("error: passed in null argument\n"); - free(resPtrs); - free(resSizes); - return results; - } - if(!resPtrs || !resSizes) { - results.error = 32; - DISPLAY("error: allocation failed\n"); - free(resPtrs); - free(resSizes); - return results; - } - - /* so resPtr is continuous */ - resPtrs[0] = malloc(srcSize); - - if(!(resPtrs[0])) { - results.error = 32; - DISPLAY("error: allocation failed\n"); - free(resPtrs); - free(resSizes); - return results; - } - - for(i = 1; i < nbBlocks; i++) { - resPtrs[i] = (void*)(((char*)resPtrs[i-1]) + srcSizes[i-1]); - } - - /* allocate own dst if NULL */ - if(dstPtrs == NULL) { - freeDST = 1; - dstPtrs = malloc(nbBlocks * sizeof(void*)); - dstCapacityToSizes = malloc(nbBlocks * sizeof(size_t)); - if(dstPtrs == NULL) { - results.error = 33; - DISPLAY("error: allocation failed\n"); - free(resPtrs); - free(resSizes); - return results; - } - - if(mode == BMK_decodeOnly) { //dst is src - size_t dstSize = 0; - for(i = 0; i < nbBlocks; i++) { - dstCapacityToSizes[i] = ZSTD_getDecompressedSize(srcPtrs[i], srcSizes[i]); - dstSize += dstCapacityToSizes[i]; - } - dstPtrs[0] = malloc(dstSize); - if(dstPtrs[0] == NULL) { - results.error = 34; - DISPLAY("error: allocation failed\n"); - goto _cleanUp; - } - for(i = 1; i < nbBlocks; i++) { - dstPtrs[i] = (void*)(((char*)dstPtrs[i-1]) + ZSTD_getDecompressedSize(srcPtrs[i-1], srcSizes[i-1])); - } - } else { - dstPtrs[0] = malloc(ZSTD_compressBound(srcSize) + (nbBlocks * 1024)); - if(dstPtrs[0] == NULL) { - results.error = 35; - DISPLAY("error: allocation failed\n"); - goto _cleanUp; - } - dstCapacityToSizes[0] = ZSTD_compressBound(srcSizes[0]); - for(i = 1; i < nbBlocks; i++) { - dstPtrs[i] = (void*)(((char*)dstPtrs[i-1]) + dstCapacityToSizes[i-1]); - dstCapacityToSizes[i] = ZSTD_compressBound(srcSizes[i]); - } - } - } - /* warmimg up memory */ - for(i = 0; i < nbBlocks; i++) { - RDG_genBuffer(dstPtrs[i], dstCapacityToSizes[i], 0.10, 0.50, 1); + /* can't do this if decode only */ + for(i = 0; i < buf.nbBlocks; i++) { + if(mode != BMK_decodeOnly) { + RDG_genBuffer(dstPtrs[i], dstCapacities[i], 0.10, 0.50, 1); + } else { + RDG_genBuffer(resPtrs[i], resSizes[i], 0.10, 0.50, 1); + } } /* Bench */ - { - { - BMK_initCCtxArgs cctxprep; - BMK_initDCtxArgs dctxprep; - cctxprep.ctx = ctx; - cctxprep.dictBuffer = dictBuffer; - cctxprep.dictBufferSize = dictBufferSize; - cctxprep.cLevel = cLevel; - cctxprep.comprParams = comprParams; - cctxprep.adv = &adv; - dctxprep.dctx = dctx; - dctxprep.dictBuffer = dictBuffer; - dctxprep.dictBufferSize = dictBufferSize; - if(loopMode == BMK_timeMode) { - BMK_customTimedReturn_t intermediateResultCompress; - BMK_customTimedReturn_t intermediateResultDecompress; - BMK_timedFnState_t* timeStateCompress = BMK_createTimeState(nbSeconds); - BMK_timedFnState_t* timeStateDecompress = BMK_createTimeState(nbSeconds); - if(mode == BMK_compressOnly) { - intermediateResultCompress.completed = 0; - intermediateResultDecompress.completed = 1; - } else if (mode == BMK_decodeOnly) { - intermediateResultCompress.completed = 1; - intermediateResultDecompress.completed = 0; - } else { /* both */ - intermediateResultCompress.completed = 0; - intermediateResultDecompress.completed = 0; + + { + /* init args */ + BMK_initCCtxArgs cctxprep; + BMK_initDCtxArgs dctxprep; + cctxprep.ctx = cctx; + cctxprep.dictBuffer = dictBuffer; + cctxprep.dictBufferSize = dictBufferSize; + cctxprep.cLevel = cLevel; + cctxprep.comprParams = comprParams; + cctxprep.adv = &adv; + dctxprep.dctx = dctx; + dctxprep.dictBuffer = dictBuffer; + dctxprep.dictBufferSize = dictBufferSize; + + if(loopMode == BMK_timeMode) { + BMK_customTimedReturn_t intermediateResultCompress; + BMK_customTimedReturn_t intermediateResultDecompress; + BMK_timedFnState_t* timeStateCompress = BMK_createTimeState(nbSeconds); + BMK_timedFnState_t* timeStateDecompress = BMK_createTimeState(nbSeconds); + if(mode == BMK_compressOnly) { + intermediateResultCompress.completed = 0; + intermediateResultDecompress.completed = 1; + } else if (mode == BMK_decodeOnly) { + intermediateResultCompress.completed = 1; + intermediateResultDecompress.completed = 0; + } else { /* both */ + intermediateResultCompress.completed = 0; + intermediateResultDecompress.completed = 0; + } + + while(!intermediateResultCompress.completed) { + intermediateResultCompress = BMK_benchFunctionTimed(timeStateCompress, &local_defaultCompress, (void*)cctx, &local_initCCtx, (void*)&cctxprep, + nbBlocks, srcPtrs, srcSizes, dstPtrs, dstCapacities, dstSizes); + + if(intermediateResultCompress.result.error) { + results.error = intermediateResultCompress.result.error; + BMK_freeTimeState(timeStateCompress); + BMK_freeTimeState(timeStateDecompress); + return results; } - while(!(intermediateResultCompress.completed && intermediateResultDecompress.completed)) { - if(!intermediateResultCompress.completed) { - intermediateResultCompress = BMK_benchFunctionTimed(timeStateCompress, &local_defaultCompress, (void*)ctx, &local_initCCtx, (void*)&cctxprep, - nbBlocks, srcPtrs, srcSizes, dstPtrs, dstCapacityToSizes); - if(intermediateResultCompress.result.error) { - results.error = intermediateResultCompress.result.error; - BMK_freeTimeState(timeStateCompress); - BMK_freeTimeState(timeStateDecompress); - goto _cleanUp; - } - results.result.cSpeed = ((double)srcSize / intermediateResultCompress.result.result.nanoSecPerRun) * 1000000000; - results.result.cSize = intermediateResultCompress.result.result.sumOfReturn; - } + results.result.cSpeed = ((double)srcSize / intermediateResultCompress.result.result.nanoSecPerRun) * TIMELOOP_NANOSEC; + results.result.cSize = intermediateResultCompress.result.result.sumOfReturn; + } - if(!intermediateResultDecompress.completed) { - if(mode == BMK_decodeOnly) { - intermediateResultDecompress = BMK_benchFunctionTimed(timeStateDecompress, &local_defaultDecompress, (void*)(dctx), &local_initDCtx, (void*)&dctxprep, - nbBlocks, (const void* const*)srcPtrs, srcSizes, dstPtrs, dstCapacityToSizes); - } else { /* both, decompressed result already written to dstPtr */ - intermediateResultDecompress = BMK_benchFunctionTimed(timeStateDecompress, &local_defaultDecompress, (void*)(dctx), &local_initDCtx, (void*)&dctxprep, - nbBlocks, (const void* const*)dstPtrs, dstCapacityToSizes, resPtrs, resSizes); - } + while(!intermediateResultDecompress.completed) { + intermediateResultDecompress = BMK_benchFunctionTimed(timeStateDecompress, &local_defaultDecompress, (void*)(dctx), &local_initDCtx, (void*)&dctxprep, + nbBlocks, (const void* const*)dstPtrs, dstSizes, resPtrs, resSizes, NULL); - if(intermediateResultDecompress.result.error) { - results.error = intermediateResultDecompress.result.error; - BMK_freeTimeState(timeStateCompress); - BMK_freeTimeState(timeStateDecompress); - goto _cleanUp; - } - results.result.dSpeed = ((double)srcSize / intermediateResultDecompress.result.result.nanoSecPerRun) * 1000000000; - } + if(intermediateResultDecompress.result.error) { + results.error = intermediateResultDecompress.result.error; + BMK_freeTimeState(timeStateCompress); + BMK_freeTimeState(timeStateDecompress); + return results; } - BMK_freeTimeState(timeStateCompress); - BMK_freeTimeState(timeStateDecompress); - } else { //iterMode; - if(mode != BMK_decodeOnly) { + results.result.dSpeed = ((double)srcSize / intermediateResultDecompress.result.result.nanoSecPerRun) * TIMELOOP_NANOSEC; + } - BMK_customReturn_t compressionResults = BMK_benchFunction(&local_defaultCompress, (void*)ctx, &local_initCCtx, (void*)&cctxprep, - nbBlocks, srcPtrs, srcSizes, dstPtrs, dstCapacityToSizes, nbSeconds); - if(compressionResults.error) { - results.error = compressionResults.error; - goto _cleanUp; - } - if(compressionResults.result.nanoSecPerRun == 0) { - results.result.cSpeed = 0; - } else { - results.result.cSpeed = (double)srcSize / compressionResults.result.nanoSecPerRun * TIMELOOP_NANOSEC; - } - results.result.cSize = compressionResults.result.sumOfReturn; + BMK_freeTimeState(timeStateCompress); + BMK_freeTimeState(timeStateDecompress); + + } else { //iterMode; + if(mode != BMK_decodeOnly) { + + BMK_customReturn_t compressionResults = BMK_benchFunction(&local_defaultCompress, (void*)cctx, &local_initCCtx, (void*)&cctxprep, + nbBlocks, srcPtrs, srcSizes, dstPtrs, dstCapacities, dstSizes, nbSeconds); + if(compressionResults.error) { + results.error = compressionResults.error; + return results; } - if(mode != BMK_compressOnly) { - BMK_customReturn_t decompressionResults; - if(mode == BMK_decodeOnly) { - decompressionResults = BMK_benchFunction( - &local_defaultDecompress, (void*)(dctx), - &local_initDCtx, (void*)&dctxprep, nbBlocks, - (const void* const*)srcPtrs, srcSizes, dstPtrs, dstCapacityToSizes, - nbSeconds); - } else { - decompressionResults = BMK_benchFunction( - &local_defaultDecompress, (void*)(dctx), - &local_initDCtx, (void*)&dctxprep, nbBlocks, - (const void* const*)dstPtrs, dstCapacityToSizes, resPtrs, resSizes, - nbSeconds); - } + if(compressionResults.result.nanoSecPerRun == 0) { + results.result.cSpeed = 0; + } else { + results.result.cSpeed = (double)srcSize / compressionResults.result.nanoSecPerRun * TIMELOOP_NANOSEC; + } + results.result.cSize = compressionResults.result.sumOfReturn; + } - if(decompressionResults.error) { - results.error = decompressionResults.error; - goto _cleanUp; - } + if(mode != BMK_compressOnly) { + BMK_customReturn_t decompressionResults; + decompressionResults = BMK_benchFunction( + &local_defaultDecompress, (void*)(dctx), + &local_initDCtx, (void*)&dctxprep, nbBlocks, + (const void* const*)dstPtrs, dstSizes, resPtrs, resSizes, NULL, + nbSeconds); - if(decompressionResults.result.nanoSecPerRun == 0) { - results.result.dSpeed = 0; - } else { - results.result.dSpeed = (double)srcSize / decompressionResults.result.nanoSecPerRun * TIMELOOP_NANOSEC; - } + if(decompressionResults.error) { + results.error = decompressionResults.error; + return results; + } + + if(decompressionResults.result.nanoSecPerRun == 0) { + results.result.dSpeed = 0; + } else { + results.result.dSpeed = (double)srcSize / decompressionResults.result.nanoSecPerRun * TIMELOOP_NANOSEC; } } } - } /* Bench */ - results.result.cMem = (1 << (comprParams->windowLog)) + ZSTD_sizeof_CCtx(ctx); - -_cleanUp: - free(resPtrs[0]); - free(resPtrs); - free(resSizes); - if(freeDST) { - free(dstPtrs[0]); - free(dstPtrs); } + /* Bench */ + results.result.cMem = (1 << (comprParams->windowLog)) + ZSTD_sizeof_CCtx(cctx); return results; } /* global winner used for display. */ //Should be totally 0 initialized? -static winnerInfo_t g_winner; +static winnerInfo_t g_winner = { { 0, 0, (size_t)-1, (size_t)-1 } , { 0, 0, 0, 0, 0, 0, ZSTD_fast } }; static constraint_t g_targetConstraints; static void BMK_printWinner(FILE* f, const U32 cLevel, const BMK_result_t result, const ZSTD_compressionParameters params, const size_t srcSize) { - if(DEBUG || (objective_lt(g_winner.result, result) && feasible(result, g_targetConstraints))) { + if(DEBUG || compareResultLT(g_winner.result, result, g_targetConstraints, srcSize)) { char lvlstr[15] = "Custom Level"; const U64 time = UTIL_clockSpanNano(g_time); const U64 minutes = time / (60ULL * TIMELOOP_NANOSEC); - if(DEBUG && (objective_lt(g_winner.result, result) && feasible(result, g_targetConstraints))) { + + if(DEBUG && compareResultLT(g_winner.result, result, g_targetConstraints, srcSize)) { DISPLAY("New Winner: \n"); } @@ -722,27 +657,24 @@ static void BMK_printWinner(FILE* f, const U32 cLevel, const BMK_result_t result fprintf(f," {%3u,%3u,%3u,%3u,%3u,%3u, %s }, ", params.windowLog, params.chainLog, params.hashLog, params.searchLog, params.searchLength, params.targetLength, g_stratName[(U32)(params.strategy)]); + if(cLevel != CUSTOM_LEVEL) { snprintf(lvlstr, 15, " Level %2u ", cLevel); } + fprintf(f, "/* %s */ /* R:%5.3f at %5.1f MB/s - %5.1f MB/s */", - lvlstr, (double)srcSize / result.cSize, result.cSpeed / 1000000., result.dSpeed / 1000000.); + lvlstr, (double)srcSize / result.cSize, result.cSpeed / (1 << 20), result.dSpeed / (1 << 20)); if(TIMED) { fprintf(f, " - %1lu:%2lu:%05.2f", (unsigned long) minutes / 60,(unsigned long) minutes % 60, (double)(time - minutes * TIMELOOP_NANOSEC * 60ULL)/TIMELOOP_NANOSEC); } fprintf(f, "\n"); - if(objective_lt(g_winner.result, result) && feasible(result, g_targetConstraints)) { + + if(compareResultLT(g_winner.result, result, g_targetConstraints, srcSize)) { BMK_translateAdvancedParams(params); g_winner.result = result; g_winner.params = params; } } - //else { - // DISPLAY("G_WINNER: "); - // DISPLAY("/* R:%5.3f at %5.1f MB/s - %5.1f MB/s */ \n",(double)srcSize / g_winner.result.cSize , g_winner.result.cSpeed / 1000000 , g_winner.result.dSpeed / 1000000); - // DISPLAY("LOSER : "); - // DISPLAY("/* R:%5.3f at %5.1f MB/s - %5.1f MB/s */ \n",(double)srcSize / result.cSize, result.cSpeed / 1000000 , result.dSpeed / 1000000); - //} } static void BMK_printWinners2(FILE* f, const winnerInfo_t* winners, size_t srcSize) @@ -903,23 +835,23 @@ static int BMK_seed(winnerInfo_t* winners, const ZSTD_compressionParameters para static ZSTD_compressionParameters sanitizeParams(ZSTD_compressionParameters params) { if (params.strategy == ZSTD_fast) - g_params.chainLog = 0, g_params.searchLog = 0; + params.chainLog = 0, params.searchLog = 0; if (params.strategy == ZSTD_dfast) - g_params.searchLog = 0; + params.searchLog = 0; if (params.strategy != ZSTD_btopt && params.strategy != ZSTD_btultra && params.strategy != ZSTD_fast) - g_params.targetLength = 0; + params.targetLength = 0; return params; } /* new length */ /* keep old array, will need if iter over strategy. */ -static int sanitizeVarArray(const int varLength, const U32* varArray, U32* varNew, const ZSTD_strategy strat) { +static int sanitizeVarArray(varInds_t* varNew, const int varLength, const varInds_t* varArray, const ZSTD_strategy strat) { int i, j = 0; for(i = 0; i < varLength; i++) { - if( !((varArray[i] == CLOG_IND && strat == ZSTD_fast) - || (varArray[i] == SLOG_IND && strat == ZSTD_dfast) - || (varArray[i] == TLEN_IND && strat != ZSTD_btopt && strat != ZSTD_btultra && strat != ZSTD_fast))) { + if( !((varArray[i] == clog_ind && strat == ZSTD_fast) + || (varArray[i] == slog_ind && strat == ZSTD_dfast) + || (varArray[i] == tlen_ind && strat != ZSTD_btopt && strat != ZSTD_btultra && strat != ZSTD_fast))) { varNew[j] = varArray[i]; j++; } @@ -930,69 +862,72 @@ static int sanitizeVarArray(const int varLength, const U32* varArray, U32* varNe /* res should be NUM_PARAMS size */ /* constructs varArray from ZSTD_compressionParameters style parameter */ -static int variableParams(const ZSTD_compressionParameters paramConstraints, U32* res) { +static int variableParams(const ZSTD_compressionParameters paramConstraints, varInds_t* res) { int j = 0; if(!paramConstraints.windowLog) { - res[j] = WLOG_IND; + res[j] = wlog_ind; j++; } if(!paramConstraints.chainLog) { - res[j] = CLOG_IND; + res[j] = clog_ind; j++; } if(!paramConstraints.hashLog) { - res[j] = HLOG_IND; + res[j] = hlog_ind; j++; } if(!paramConstraints.searchLog) { - res[j] = SLOG_IND; + res[j] = slog_ind; j++; } if(!paramConstraints.searchLength) { - res[j] = SLEN_IND; + res[j] = slen_ind; j++; } if(!paramConstraints.targetLength) { - res[j] = TLEN_IND; + res[j] = tlen_ind; j++; } return j; } +/* bin-search on tlen_table for correct index. */ +static int tlen_inv(U32 x) { + int lo = 0; + int hi = TLEN_RANGE; + while(lo < hi) { + int mid = (lo + hi) / 2; + if(tlen_table[mid] < x) { + lo = mid + 1; + } if(tlen_table[mid] == x) { + return mid; + } else { + hi = mid; + } + } + return lo; +} + /* amt will probably always be \pm 1? */ /* slight change from old paramVariation, targetLength can only take on powers of 2 now (999 ~= 1024?) */ /* take max/min bounds into account as well? */ -static void paramVaryOnce(const U32 paramIndex, const int amt, ZSTD_compressionParameters* ptr) { +static void paramVaryOnce(const varInds_t paramIndex, const int amt, ZSTD_compressionParameters* ptr) { switch(paramIndex) { - case WLOG_IND: ptr->windowLog += amt; break; - case CLOG_IND: ptr->chainLog += amt; break; - case HLOG_IND: ptr->hashLog += amt; break; - case SLOG_IND: ptr->searchLog += amt; break; - case SLEN_IND: ptr->searchLength += amt; break; - case TLEN_IND: - if(amt >= 0) { - if(ptr->targetLength == 0) { - if(amt > 0) { - ptr->targetLength = MIN(1 << (amt - 1), 999); - } - } else { - ptr->targetLength <<= amt; - ptr->targetLength = MIN(ptr->targetLength, 999); - } - } else { - if(ptr->targetLength == 999) { - ptr->targetLength = 1024; - } - ptr->targetLength >>= -amt; - } + case wlog_ind: ptr->windowLog += amt; break; + case clog_ind: ptr->chainLog += amt; break; + case hlog_ind: ptr->hashLog += amt; break; + case slog_ind: ptr->searchLog += amt; break; + case slen_ind: ptr->searchLength += amt; break; + case tlen_ind: + ptr->targetLength = tlen_table[MAX(0, MIN(TLEN_RANGE - 1, tlen_inv(ptr->targetLength) + amt))]; break; default: break; } } /* varies ptr by nbChanges respecting varyParams*/ -static void paramVariation(ZSTD_compressionParameters* ptr, const U32* varyParams, const int varyLen, const U32 nbChanges) +static void paramVariation(ZSTD_compressionParameters* ptr, const varInds_t* varyParams, const int varyLen, const U32 nbChanges) { ZSTD_compressionParameters p; U32 validated = 0; @@ -1009,7 +944,7 @@ static void paramVariation(ZSTD_compressionParameters* ptr, const U32* varyParam } /* length of memo table given free variables */ -static size_t memoTableLen(const U32* varyParams, const int varyLen) { +static size_t memoTableLen(const varInds_t* varyParams, const int varyLen) { size_t arrayLen = 1; int i; for(i = 0; i < varyLen; i++) { @@ -1018,53 +953,45 @@ static size_t memoTableLen(const U32* varyParams, const int varyLen) { return arrayLen; } -//sort of ~lg2 (replace 1024 w/ 999, and add 0 at lower end of range) for memoTableInd Tlen -static unsigned lg2(unsigned x) { - if(x == 999) { - return 11; - } - return x ? ZSTD_highbit32(x) + 1 : 0; -} - /* returns unique index of compression parameters */ -static unsigned memoTableInd(const ZSTD_compressionParameters* ptr, const U32* varyParams, const int varyLen) { +static unsigned memoTableInd(const ZSTD_compressionParameters* ptr, const varInds_t* varyParams, const int varyLen) { int i; unsigned ind = 0; for(i = 0; i < varyLen; i++) { switch(varyParams[i]) { - case WLOG_IND: ind *= WLOG_RANGE; ind += ptr->windowLog - ZSTD_WINDOWLOG_MIN ; break; - case CLOG_IND: ind *= CLOG_RANGE; ind += ptr->chainLog - ZSTD_CHAINLOG_MIN ; break; - case HLOG_IND: ind *= HLOG_RANGE; ind += ptr->hashLog - ZSTD_HASHLOG_MIN ; break; - case SLOG_IND: ind *= SLOG_RANGE; ind += ptr->searchLog - ZSTD_SEARCHLOG_MIN ; break; - case SLEN_IND: ind *= SLEN_RANGE; ind += ptr->searchLength - ZSTD_SEARCHLENGTH_MIN; break; - case TLEN_IND: ind *= TLEN_RANGE; ind += lg2(ptr->targetLength) - ZSTD_TARGETLENGTH_MIN; break; + case wlog_ind: ind *= WLOG_RANGE; ind += ptr->windowLog - ZSTD_WINDOWLOG_MIN ; break; + case clog_ind: ind *= CLOG_RANGE; ind += ptr->chainLog - ZSTD_CHAINLOG_MIN ; break; + case hlog_ind: ind *= HLOG_RANGE; ind += ptr->hashLog - ZSTD_HASHLOG_MIN ; break; + case slog_ind: ind *= SLOG_RANGE; ind += ptr->searchLog - ZSTD_SEARCHLOG_MIN ; break; + case slen_ind: ind *= SLEN_RANGE; ind += ptr->searchLength - ZSTD_SEARCHLENGTH_MIN; break; + case tlen_ind: ind *= TLEN_RANGE; ind += tlen_inv(ptr->targetLength) - ZSTD_TARGETLENGTH_MIN; break; } } return ind; } /* inverse of above function (from index to parameters) */ -static void memoTableIndInv(ZSTD_compressionParameters* ptr, const U32* varyParams, const int varyLen, size_t ind) { +static void memoTableIndInv(ZSTD_compressionParameters* ptr, const varInds_t* varyParams, const int varyLen, size_t ind) { int i; for(i = varyLen - 1; i >= 0; i--) { switch(varyParams[i]) { - case WLOG_IND: ptr->windowLog = ind % WLOG_RANGE + ZSTD_WINDOWLOG_MIN; ind /= WLOG_RANGE; break; - case CLOG_IND: ptr->chainLog = ind % CLOG_RANGE + ZSTD_CHAINLOG_MIN; ind /= CLOG_RANGE; break; - case HLOG_IND: ptr->hashLog = ind % HLOG_RANGE + ZSTD_HASHLOG_MIN; ind /= HLOG_RANGE; break; - case SLOG_IND: ptr->searchLog = ind % SLOG_RANGE + ZSTD_SEARCHLOG_MIN; ind /= SLOG_RANGE; break; - case SLEN_IND: ptr->searchLength = ind % SLEN_RANGE + ZSTD_SEARCHLENGTH_MIN; ind /= SLEN_RANGE; break; - case TLEN_IND: ptr->targetLength = (ind % TLEN_RANGE) ? MIN(1 << ((ind % TLEN_RANGE) - 1), 999) : 0; ind /= TLEN_RANGE; break; + case wlog_ind: ptr->windowLog = ind % WLOG_RANGE + ZSTD_WINDOWLOG_MIN; ind /= WLOG_RANGE; break; + case clog_ind: ptr->chainLog = ind % CLOG_RANGE + ZSTD_CHAINLOG_MIN; ind /= CLOG_RANGE; break; + case hlog_ind: ptr->hashLog = ind % HLOG_RANGE + ZSTD_HASHLOG_MIN; ind /= HLOG_RANGE; break; + case slog_ind: ptr->searchLog = ind % SLOG_RANGE + ZSTD_SEARCHLOG_MIN; ind /= SLOG_RANGE; break; + case slen_ind: ptr->searchLength = ind % SLEN_RANGE + ZSTD_SEARCHLENGTH_MIN; ind /= SLEN_RANGE; break; + case tlen_ind: ptr->targetLength = tlen_table[(ind % TLEN_RANGE)]; ind /= TLEN_RANGE; break; } } } - /* Initialize memotable, immediately mark redundant / obviously infeasible params as */ -static void memoTableInit(U8* memoTable, ZSTD_compressionParameters paramConstraints, const constraint_t target, const U32* varyParams, const int varyLen, const size_t srcSize) { +static void memoTableInit(U8* memoTable, ZSTD_compressionParameters paramConstraints, const constraint_t target, const varInds_t* varyParams, const int varyLen, const size_t srcSize) { size_t i; size_t arrayLen = memoTableLen(varyParams, varyLen); int cwFixed = !paramConstraints.chainLog || !paramConstraints.windowLog; int scFixed = !paramConstraints.searchLog || !paramConstraints.chainLog; + int whFixed = !paramConstraints.windowLog || !paramConstraints.hashLog; int wFixed = !paramConstraints.windowLog; int j = 0; memset(memoTable, 0, arrayLen); @@ -1076,7 +1003,7 @@ static void memoTableInit(U8* memoTable, ZSTD_compressionParameters paramConstra memoTable[i] = 255; j++; } - if(wFixed && (1ULL << paramConstraints.windowLog) > (srcSize << 1)) { + if(wFixed && (1ULL << paramConstraints.windowLog) > (srcSize << 2)) { memoTable[i] = 255; } /* nil out parameter sets equivalent to others. */ @@ -1093,12 +1020,20 @@ static void memoTableInit(U8* memoTable, ZSTD_compressionParameters paramConstra } } } + if(scFixed) { if(paramConstraints.searchLog > paramConstraints.chainLog) { if(memoTable[i] != 255) { j++; } memoTable[i] = 255; } } + + if(whFixed) { + if(paramConstraints.hashLog > paramConstraints.windowLog + 1) { + if(memoTable[i] != 255) { j++; } + memoTable[i] = 255; + } + } } DEBUGOUTPUT("%d / %d Invalid\n", j, (int)i); if((int)i == j) { @@ -1106,27 +1041,7 @@ static void memoTableInit(U8* memoTable, ZSTD_compressionParameters paramConstra } } -/* inits memotables for all (including mallocs), all strategies */ -/* takes unsanitized varyParams */ -static U8** memoTableInitAll(ZSTD_compressionParameters paramConstraints, constraint_t target, const U32* varyParams, const int varyLen, const size_t srcSize) { - U32 varNew[NUM_PARAMS]; - int varLenNew; - U8** mtAll = malloc(sizeof(U8*) * (ZSTD_btultra + 1)); - int i; - if(mtAll == NULL) { - return NULL; - } - for(i = 1; i <= (int)ZSTD_btultra; i++) { - varLenNew = sanitizeVarArray(varyLen, varyParams, varNew, i); - mtAll[i] = malloc(sizeof(U8) * memoTableLen(varNew, varLenNew)); - if(mtAll[i] == NULL) { - return NULL; - } - memoTableInit(mtAll[i], paramConstraints, target, varNew, varLenNew, srcSize); - } - return mtAll; -} - +/* frees all allocated memotables */ static void memoTableFreeAll(U8** mtAll) { int i; if(mtAll == NULL) { return; } @@ -1136,6 +1051,28 @@ static void memoTableFreeAll(U8** mtAll) { free(mtAll); } +/* inits memotables for all (including mallocs), all strategies */ +/* takes unsanitized varyParams */ +static U8** memoTableInitAll(ZSTD_compressionParameters paramConstraints, constraint_t target, const varInds_t* varyParams, const int varyLen, const size_t srcSize) { + varInds_t varNew[NUM_PARAMS]; + int varLenNew; + U8** mtAll = calloc(sizeof(U8*),(ZSTD_btultra + 1)); + int i; + if(mtAll == NULL) { + return NULL; + } + for(i = 1; i <= (int)ZSTD_btultra; i++) { + varLenNew = sanitizeVarArray(varNew, varyLen, varyParams, i); + mtAll[i] = malloc(sizeof(U8) * memoTableLen(varNew, varLenNew)); + if(mtAll[i] == NULL) { + memoTableFreeAll(mtAll); + return NULL; + } + memoTableInit(mtAll[i], paramConstraints, target, varNew, varLenNew, srcSize); + } + return mtAll; +} + #define PARAMTABLELOG 25 #define PARAMTABLESIZE (1< 0 && tries > 0); memoTableIndInv(pc, varArray, varLen, (unsigned)ind); - *pc = sanitizeParams(*pc); + //*pc = sanitizeParams(*pc); } static void BMK_selectRandomStart( @@ -1250,7 +1187,7 @@ static void BMK_benchFullTable(ZSTD_CCtx* cctx, ZSTD_DCtx* dctx, const void* src winnerInfo_t winners[NB_LEVELS_TRACKED+1]; const char* const rfName = "grillResults.txt"; FILE* const f = fopen(rfName, "w"); - const size_t blockSize = g_blockSize ? g_blockSize : srcSize; /* cut by block or not ? */ + const size_t blockSize = g_blockSize ? g_blockSize : ZSTD_BLOCKSIZE_MAX; /* cut by block or not ? */ /* init */ assert(g_singleRun==0); @@ -1390,574 +1327,204 @@ int benchFiles(const char** fileNamesTable, int nbFiles) return 0; } - - -/* -checks feasibility with uncertainty. --1 : certainly infeasible - 0 : uncertain - 1 : certainly feasible -*/ -static int uncertainFeasibility(double const uncertaintyConstantC, double const uncertaintyConstantD, const constraint_t paramTarget, const BMK_result_t* const results) { - if((paramTarget.cSpeed != 0 && results->cSpeed * uncertaintyConstantC < paramTarget.cSpeed) || - (paramTarget.dSpeed != 0 && results->dSpeed * uncertaintyConstantD < paramTarget.dSpeed) || - (paramTarget.cMem != 0 && results->cMem > paramTarget.cMem)) { - return -1; - } else if((paramTarget.cSpeed == 0 || results->cSpeed / uncertaintyConstantC > paramTarget.cSpeed) && - (paramTarget.dSpeed == 0 || results->dSpeed / uncertaintyConstantD > paramTarget.dSpeed) && - (paramTarget.cMem == 0 || results->cMem <= paramTarget.cMem)) { - return 1; - } else { - return 0; - } -} - -/* 1 - better than prev best - 0 - uncertain - -1 - worse - assume prev_best status is run fully? - but then we'd have to rerun any winners anyway */ -/* not as useful as initially believed */ -static int uncertainComparison(double const uncertaintyConstantC, double const uncertaintyConstantD, const BMK_result_t* candidate, const BMK_result_t* prevBest) { - (void)uncertaintyConstantD; //unused for now - if(candidate->cSpeed > prevBest->cSpeed * uncertaintyConstantC) { - return 1; - } else if (candidate->cSpeed * uncertaintyConstantC < prevBest->cSpeed) { - return -1; - } else { - return 0; - } -} - /*benchmarks and tests feasibility together 1 = true = better 0 = false = not better if true then resultPtr will give results. 2+ on error? */ -//Maybe use compress_only for benchmark -#define INFEASIBLE_RESULT 0 -#define FEASIBLE_RESULT 1 +//Maybe use compress_only for benchmark first run? +#define WORSE_RESULT 0 +#define BETTER_RESULT 1 #define ERROR_RESULT 2 -static int feasibleBench(BMK_result_t* resultPtr, - const void* const * const srcPtrs, size_t const * const srcSizes, - void** const dstPtrs, size_t* dstCapacityToSizes, U32 const nbBlocks, - void* dictBuffer, const size_t dictBufferSize, - ZSTD_CCtx* ctx, ZSTD_DCtx* dctx, - const ZSTD_compressionParameters cParams, - const constraint_t target, - BMK_result_t* winnerResult) { - BMK_return_t benchres; - U64 loopDurationC = 0, loopDurationD = 0; - double uncertaintyConstantC, uncertaintyConstantD; - size_t srcSize = 0; - U32 i; - //alternative - test 1 iter for ratio, (possibility of error 3 which is fine), - //maybe iter this until 2x measurable for better guarantee? - DEBUGOUTPUT("Feas:\n"); - benchres = BMK_benchMemInvertible(srcPtrs, srcSizes, dstPtrs, dstCapacityToSizes, nbBlocks, 0, &cParams, dictBuffer, dictBufferSize, ctx, dctx, - BMK_both, BMK_iterMode, 1); - if(benchres.error) { - DISPLAY("ERROR %d !!\n", benchres.error); - } - for(i = 0; i < nbBlocks; i++) { - srcSize += srcSizes[i]; - } - BMK_printWinner(stdout, CUSTOM_LEVEL, benchres.result, cParams, srcSize); +//add worse result complete for worse results of length > 1 sec? - if(!benchres.error) { - *resultPtr = benchres.result; - /* if speed is 0 (only happens when time = 0) */ - if(eqZero(benchres.result.cSpeed)) { - loopDurationC = 0; - uncertaintyConstantC = 2; - } else { - loopDurationC = (U64)((double)(srcSize * TIMELOOP_NANOSEC) / benchres.result.cSpeed); - //problem - tested in fullbench, saw speed vary 3x between iters, maybe raise uncertaintyConstraint up? - //possibly has to do with initCCtx? or system stuff? - //asymmetric +/- constant needed? - uncertaintyConstantC = MIN((loopDurationC + (double)(2 * g_clockGranularity)/loopDurationC) * 1.1, 3); //.02 seconds - } - if(eqZero(benchres.result.dSpeed)) { - loopDurationD = 0; - uncertaintyConstantD = 2; - } else { - loopDurationD = (U64)((double)(srcSize * TIMELOOP_NANOSEC) / benchres.result.dSpeed); - //problem - tested in fullbench, saw speed vary 3x between iters, maybe raise uncertaintyConstraint up? - //possibly has to do with initCCtx? or system stuff? - //asymmetric +/- constant needed? - uncertaintyConstantD = MIN((loopDurationD + (double)(2 * g_clockGranularity)/loopDurationD) * 1.1, 3); //.02 seconds - } - - - if(benchres.result.cSize < winnerResult->cSize) { //better compression ratio, just needs to be feasible - int feas; - if(loopDurationC < TIMELOOP_NANOSEC / 10) { - BMK_return_t benchres2 = BMK_benchMemInvertible(srcPtrs, srcSizes, dstPtrs, dstCapacityToSizes, nbBlocks, 0, &cParams, dictBuffer, dictBufferSize, ctx, dctx, - BMK_compressOnly, BMK_iterMode, 1); - if(benchres2.error) { - return ERROR_RESULT; - } else { - benchres = benchres2; - } - } - if(loopDurationD < TIMELOOP_NANOSEC / 10) { - BMK_return_t benchres2 = BMK_benchMemInvertible((const void* const*)dstPtrs, dstCapacityToSizes, NULL, NULL, nbBlocks, 0, &cParams, dictBuffer, dictBufferSize, ctx, dctx, - BMK_decodeOnly, BMK_iterMode, 1); - if(benchres2.error) { - return ERROR_RESULT; - } else { - benchres.result.dSpeed = benchres2.result.dSpeed; - } - } - *resultPtr = benchres.result; - - feas = uncertainFeasibility(uncertaintyConstantC, uncertaintyConstantD, target, &(benchres.result)); - if(feas == 0) { // uncertain feasibility - if(loopDurationC < TIMELOOP_NANOSEC) { - BMK_return_t benchres2 = BMK_benchMemInvertible(srcPtrs, srcSizes, dstPtrs, dstCapacityToSizes, nbBlocks, 0, &cParams, dictBuffer, dictBufferSize, ctx, dctx, - BMK_compressOnly, BMK_timeMode, 1); - if(benchres2.error) { - return ERROR_RESULT; - } else { - benchres.result.cSpeed = benchres2.result.cSpeed; - } - } - if(loopDurationD < TIMELOOP_NANOSEC) { - BMK_return_t benchres2 = BMK_benchMemInvertible((const void* const*)dstPtrs, dstCapacityToSizes, NULL, NULL, nbBlocks, 0, &cParams, dictBuffer, dictBufferSize, ctx, dctx, - BMK_decodeOnly, BMK_timeMode, 1); - if(benchres2.error) { - return ERROR_RESULT; - } else { - benchres.result.dSpeed = benchres2.result.dSpeed; - } - } - *resultPtr = benchres.result; - return feasible(benchres.result, target); - } else { //feas = 1 or -1 map to 1, 0 respectively - return (feas + 1) >> 1; //relies on INFEASIBLE_RESULT == 0, FEASIBLE_RESULT == 1 - } - } else if (benchres.result.cSize == winnerResult->cSize) { //equal ratio, needs to be better than winner in cSpeed/ dSpeed / cMem - int feas; - if(loopDurationC < TIMELOOP_NANOSEC / 10) { - BMK_return_t benchres2 = BMK_benchMemInvertible(srcPtrs, srcSizes, dstPtrs, dstCapacityToSizes, nbBlocks, 0, &cParams, dictBuffer, dictBufferSize, ctx, dctx, - BMK_compressOnly, BMK_iterMode, 1); - if(benchres2.error) { - return ERROR_RESULT; - } else { - benchres = benchres2; - } - } - if(loopDurationD < TIMELOOP_NANOSEC / 10) { - BMK_return_t benchres2 = BMK_benchMemInvertible((const void* const*)dstPtrs, dstCapacityToSizes, NULL, NULL, nbBlocks, 0, &cParams, dictBuffer, dictBufferSize, ctx, dctx, - BMK_decodeOnly, BMK_iterMode, 1); - if(benchres2.error) { - return ERROR_RESULT; - } else { - benchres.result.dSpeed = benchres2.result.dSpeed; - } - } - feas = uncertainFeasibility(uncertaintyConstantC, uncertaintyConstantD, target, &(benchres.result)); - if(feas == 0) { // uncertain feasibility - if(loopDurationC < TIMELOOP_NANOSEC) { - BMK_return_t benchres2 = BMK_benchMemInvertible(srcPtrs, srcSizes, dstPtrs, dstCapacityToSizes, nbBlocks, 0, &cParams, dictBuffer, dictBufferSize, ctx, dctx, - BMK_compressOnly, BMK_timeMode, 1); - if(benchres2.error) { - return ERROR_RESULT; - } else { - benchres.result.cSpeed = benchres2.result.cSpeed; - } - } - if(loopDurationD < TIMELOOP_NANOSEC) { - BMK_return_t benchres2 = BMK_benchMemInvertible((const void* const*)dstPtrs, dstCapacityToSizes, NULL, NULL, nbBlocks, 0, &cParams, dictBuffer, dictBufferSize, ctx, dctx, - BMK_decodeOnly, BMK_timeMode, 1); - if(benchres2.error) { - return ERROR_RESULT; - } else { - benchres.result.dSpeed = benchres2.result.dSpeed; - } - } - - *resultPtr = benchres.result; - return feasible(benchres.result, target) && objective_lt(*winnerResult, benchres.result); - } else if (feas == 1) { //no need to check feasibility compares (maybe only it is chosen as a winner) - int btw = uncertainComparison(uncertaintyConstantC, uncertaintyConstantD, &(benchres.result), winnerResult); - if(btw == -1) { - return INFEASIBLE_RESULT; - } else { //possibly better, benchmark and find out - benchres = BMK_benchMemInvertible(srcPtrs, srcSizes, dstPtrs, dstCapacityToSizes, nbBlocks, 0, &cParams, dictBuffer, dictBufferSize, ctx, dctx, - BMK_both, BMK_timeMode, 1); - *resultPtr = benchres.result; - return objective_lt(*winnerResult, benchres.result); - } - } else { //feas == -1 - return INFEASIBLE_RESULT; //infeasible - } - } else { - return INFEASIBLE_RESULT; //infeasible - } - } else { - return ERROR_RESULT; //BMK error - } -} - -//same as before, but +/-? -//alternative, just return comparison result, leave caller to worry about feasibility. -//have version of benchMemAdvanced which takes in dstBuffer/cap as well? -//(motivation: repeat tests (maybe just on decompress) don't need further compress runs) -static int infeasibleBench(BMK_result_t* resultPtr, - const void* const * const srcPtrs, size_t const * const srcSizes, - void** const dstPtrs, size_t* dstCapacityToSizes, U32 const nbBlocks, - void* dictBuffer, const size_t dictBufferSize, - ZSTD_CCtx* ctx, ZSTD_DCtx* dctx, +/* variation between 2nd run and full second bmk */ +#define VARIANCE 1.1 +static int allBench(BMK_result_t* resultPtr, + buffers_t buf, contexts_t ctx, const ZSTD_compressionParameters cParams, const constraint_t target, - BMK_result_t* winnerResult) { + BMK_result_t* winnerResult, int feas) { BMK_return_t benchres; - BMK_result_t resultMin, resultMax; + BMK_result_t resultMax; U64 loopDurationC = 0, loopDurationD = 0; double uncertaintyConstantC, uncertaintyConstantD; double winnerRS; - size_t srcSize = 0; - U32 i; - benchres = BMK_benchMemInvertible(srcPtrs, srcSizes, dstPtrs, dstCapacityToSizes, nbBlocks, 0, &cParams, dictBuffer, dictBufferSize, ctx, dctx, - BMK_both, BMK_iterMode, 1); - for(i = 0; i < nbBlocks; i++) { - srcSize += srcSizes[i]; - } - BMK_printWinner(stdout, CUSTOM_LEVEL, benchres.result, cParams, srcSize); - winnerRS = resultScore(*winnerResult, srcSize, target); + /* initial benchmarking, gives exact ratio and memory, warms up future runs */ + benchres = BMK_benchMemInvertible(buf, ctx, 0, &cParams, BMK_both, BMK_iterMode, 1); + winnerRS = resultScore(*winnerResult, buf.srcSize, target); DEBUGOUTPUT("WinnerScore: %f\n ", winnerRS); - if(!benchres.error) { - *resultPtr = benchres.result; - if(eqZero(benchres.result.cSpeed)) { - loopDurationC = 0; - uncertaintyConstantC = 2; - } else { - loopDurationC = (U64)((double)(srcSize * TIMELOOP_NANOSEC) / benchres.result.cSpeed); - uncertaintyConstantC = MIN((loopDurationC + (double)(2 * g_clockGranularity)/loopDurationC * 1.1), 3); //.02 seconds - } + if(benchres.error) { + DEBUGOUTPUT("Benchmarking failed\n"); + return ERROR_RESULT; + } + *resultPtr = benchres.result; - if(eqZero(benchres.result.dSpeed)) { - loopDurationD = 0; - uncertaintyConstantD = 2; - } else { - loopDurationD = (U64)((double)(srcSize * TIMELOOP_NANOSEC) / benchres.result.dSpeed); - uncertaintyConstantD = MIN((loopDurationD + (double)(2 * g_clockGranularity)/loopDurationD) * 1.1 , 3); //.02 seconds - } + /* calculate uncertainty in compression / decompression runs */ + if(eqZero(benchres.result.cSpeed)) { + loopDurationC = 0; + uncertaintyConstantC = 3; + } else { + loopDurationC = (U64)((double)(buf.srcSize * TIMELOOP_NANOSEC) / benchres.result.cSpeed); + uncertaintyConstantC = ((loopDurationC + (double)(2 * g_clockGranularity))/loopDurationC) * VARIANCE; + } + if(eqZero(benchres.result.dSpeed)) { + loopDurationD = 0; + uncertaintyConstantD = 3; + } else { + loopDurationD = (U64)((double)(buf.srcSize * TIMELOOP_NANOSEC) / benchres.result.dSpeed); + uncertaintyConstantD = ((loopDurationD + (double)(2 * g_clockGranularity))/loopDurationD) * VARIANCE; + } - if(loopDurationC < TIMELOOP_NANOSEC / 10) { - BMK_return_t benchres2 = BMK_benchMemInvertible(srcPtrs, srcSizes, dstPtrs, dstCapacityToSizes, nbBlocks, 0, &cParams, dictBuffer, dictBufferSize, ctx, dctx, - BMK_compressOnly, BMK_iterMode, 1); - if(benchres2.error) { - return ERROR_RESULT; - } else { - benchres = benchres2; - } - } - if(loopDurationD < TIMELOOP_NANOSEC / 10) { - BMK_return_t benchres2 = BMK_benchMemInvertible((const void* const*)dstPtrs, dstCapacityToSizes, NULL, NULL, nbBlocks, 0, &cParams, dictBuffer, dictBufferSize, ctx, dctx, - BMK_decodeOnly, BMK_iterMode, 1); - if(benchres2.error) { - return ERROR_RESULT; - } else { - benchres.result.dSpeed = benchres2.result.dSpeed; - } - } - *resultPtr = benchres.result; + /* anything with worse ratio in feas is definitely worse, discard */ + if(feas && benchres.result.cSize < winnerResult->cSize) { + return WORSE_RESULT; + } - /* benchres's certainty range. */ - resultMax = benchres.result; - resultMin = benchres.result; - resultMax.cSpeed *= uncertaintyConstantC; - resultMax.dSpeed *= uncertaintyConstantD; - resultMin.cSpeed /= uncertaintyConstantC; - resultMin.dSpeed /= uncertaintyConstantD; - if (winnerRS > resultScore(resultMax, srcSize, target)) { - return INFEASIBLE_RESULT; - } else { - if(loopDurationC < TIMELOOP_NANOSEC) { - BMK_return_t benchres2 = BMK_benchMemInvertible(srcPtrs, srcSizes, dstPtrs, dstCapacityToSizes, nbBlocks, 0, &cParams, dictBuffer, dictBufferSize, ctx, dctx, - BMK_compressOnly, BMK_timeMode, 1); - if(benchres2.error) { - return ERROR_RESULT; - } else { - benchres.result.cSpeed = benchres2.result.cSpeed; - } - } - if(loopDurationD < TIMELOOP_NANOSEC) { - BMK_return_t benchres2 = BMK_benchMemInvertible((const void* const*)dstPtrs, dstCapacityToSizes, NULL, NULL, nbBlocks, 0, &cParams, dictBuffer, dictBufferSize, ctx, dctx, - BMK_decodeOnly, BMK_timeMode, 1); - if(benchres2.error) { - return ERROR_RESULT; - } else { - benchres.result.dSpeed = benchres2.result.dSpeed; - } - } - *resultPtr = benchres.result; - return (resultScore(benchres.result, srcSize, target) > winnerRS); + /* second run, if first run is too short, gives approximate cSpeed + dSpeed */ + if(loopDurationC < TIMELOOP_NANOSEC / 10) { + BMK_return_t benchres2 = BMK_benchMemInvertible(buf, ctx, 0, &cParams, BMK_compressOnly, BMK_iterMode, 1); + if(benchres2.error) { + return ERROR_RESULT; } + benchres = benchres2; + } + if(loopDurationD < TIMELOOP_NANOSEC / 10) { + BMK_return_t benchres2 = BMK_benchMemInvertible(buf, ctx, 0, &cParams, BMK_decodeOnly, BMK_iterMode, 1); + if(benchres2.error) { + return ERROR_RESULT; + } + benchres.result.dSpeed = benchres2.result.dSpeed; + } *resultPtr = benchres.result; - } else { - return ERROR_RESULT; //BMK error + + /* optimistic assumption of benchres.result */ + resultMax = benchres.result; + resultMax.cSpeed *= uncertaintyConstantC; + resultMax.dSpeed *= uncertaintyConstantD; + + /* disregard infeasible results in feas mode */ + /* disregard if resultMax < winner in infeas mode */ + if((feas && !feasible(resultMax, target)) || + (!feas && (winnerRS > resultScore(resultMax, buf.srcSize, target)))) { + return WORSE_RESULT; + } + + /* Final full run if estimates are unclear */ + if(loopDurationC < TIMELOOP_NANOSEC) { + BMK_return_t benchres2 = BMK_benchMemInvertible(buf, ctx, 0, &cParams, BMK_compressOnly, BMK_timeMode, 1); + if(benchres2.error) { + return ERROR_RESULT; + } + benchres.result.cSpeed = benchres2.result.cSpeed; + } + + if(loopDurationD < TIMELOOP_NANOSEC) { + BMK_return_t benchres2 = BMK_benchMemInvertible(buf, ctx, 0, &cParams, BMK_decodeOnly, BMK_timeMode, 1); + if(benchres2.error) { + return ERROR_RESULT; + } + benchres.result.dSpeed = benchres2.result.dSpeed; + } + + *resultPtr = benchres.result; + + /* compare by resultScore when in infeas */ + /* compare by compareResultLT when in feas */ + if((!feas && (resultScore(benchres.result, buf.srcSize, target) > resultScore(*winnerResult, buf.srcSize, target))) || + (feas && (compareResultLT(*winnerResult, benchres.result, target, buf.srcSize))) ) { + return BETTER_RESULT; + } else { + return WORSE_RESULT; } } /* wrap feasibleBench w/ memotable */ #define INFEASIBLE_THRESHOLD 200 -static int feasibleBenchMemo(BMK_result_t* resultPtr, - const void* srcBuffer, const size_t srcSize, - void* dstBuffer, const size_t dstSize, - void* dictBuffer, const size_t dictSize, - const size_t* fileSizes, const size_t nbFiles, - ZSTD_CCtx* ctx, ZSTD_DCtx* dctx, - const ZSTD_compressionParameters cParams, - const constraint_t target, - BMK_result_t* winnerResult, U8* memoTable, - const U32* varyParams, const int varyLen) { - - const size_t memind = memoTableInd(&cParams, varyParams, varyLen); - if(memoTable[memind] >= INFEASIBLE_THRESHOLD) { - return INFEASIBLE_RESULT; - } else { - const size_t blockSize = g_blockSize ? g_blockSize : srcSize; - U32 const maxNbBlocks = (U32) ((srcSize + (blockSize-1)) / blockSize) + (U32)nbFiles; - const void ** const srcPtrs = (const void** const)malloc(maxNbBlocks * sizeof(void*)); - size_t* const srcSizes = (size_t* const)malloc(maxNbBlocks * sizeof(size_t)); - void ** const dstPtrs = (void** const)malloc(maxNbBlocks * sizeof(void*)); - size_t* const dstCapacities = (size_t* const)malloc(maxNbBlocks * sizeof(size_t)); - U32 nbBlocks; - int res; - - if(!srcPtrs || !srcSizes || !dstPtrs || !dstCapacities) { - free(srcPtrs); - free(srcSizes); - free(dstPtrs); - free(dstCapacities); - DISPLAY("Allocation Error\n"); - return ERROR_RESULT; - } - - { - const char* srcPtr = (const char*)srcBuffer; - char* dstPtr = (char*)dstBuffer; - size_t dstSizeRemaining = dstSize; - U32 fileNb; - for (nbBlocks=0, fileNb=0; fileNb dstSizeRemaining) { - DEBUGOUTPUT("Warning: dstSize too small to benchmark completely \n"); - remaining = dstSizeRemaining; - dstSizeRemaining = 0; - } else { - dstSizeRemaining -= remaining; - } - for ( ; nbBlocks= INFEASIBLE_THRESHOLD) { - return INFEASIBLE_RESULT; //see feasibleBenchMemo for concerns - } else { - const size_t blockSize = g_blockSize ? g_blockSize : srcSize; - U32 const maxNbBlocks = (U32) ((srcSize + (blockSize-1)) / blockSize) + (U32)nbFiles; - const void ** const srcPtrs = (const void** const)malloc(maxNbBlocks * sizeof(void*)); - size_t* const srcSizes = (size_t* const)malloc(maxNbBlocks * sizeof(size_t)); - void ** const dstPtrs = (void** const)malloc(maxNbBlocks * sizeof(void*)); - size_t* const dstCapacities = (size_t* const)malloc(maxNbBlocks * sizeof(size_t)); - U32 nbBlocks; - int res; + int res; - if(!srcPtrs || !srcSizes || !dstPtrs || !dstCapacities) { - free(srcPtrs); - free(srcSizes); - free(dstPtrs); - free(dstCapacities); - DISPLAY("Allocation Error\n"); - return ERROR_RESULT; - } + if(memoTable[memind] >= INFEASIBLE_THRESHOLD) { return WORSE_RESULT; } - { - const char* srcPtr = (const char*)srcBuffer; - char* dstPtr = (char*)dstBuffer; - size_t dstSizeRemaining = dstSize; - U32 fileNb; - for (nbBlocks=0, fileNb=0; fileNb dstSizeRemaining) { - DEBUGOUTPUT("Warning: dstSize too small to benchmark completely \n"); - remaining = dstSizeRemaining; - dstSizeRemaining = 0; - } else { - dstSizeRemaining -= remaining; - } - for ( ; nbBlocks (hopefully) feasible */ - /* when nothing is found, this garbages part 2. */ { - winnerInfo_t bestFeasible1; /* uses feasibleBench Metric */ - - //init these params - bestFeasible1.params = cparam; - bestFeasible1.result.cSpeed = 0; - bestFeasible1.result.dSpeed = 0; - bestFeasible1.result.cMem = (size_t)-1; - bestFeasible1.result.cSize = (size_t)-1; + winnerInfo_t bestFeasible1 = initWinnerInfo(cparam); DISPLAY("Climb Part 1\n"); while(better) { - int i, d; + int i, dist, offset; better = 0; DEBUGOUTPUT("Start\n"); cparam = winnerInfo.params; - BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, srcSize); + BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, buf.srcSize); candidateInfo.params = cparam; //all dist-1 targets //if we early end this, we should also randomize the order these are picked. for(i = 0; i < varLen; i++) { - paramVaryOnce(varArray[i], 1, &candidateInfo.params); /* +1 */ - candidateInfo.params = sanitizeParams(candidateInfo.params); - //evaluate - if(!ZSTD_isError(ZSTD_checkCParams(candidateInfo.params))) { - //if(cParamValid(candidateInfo.params)) { - int res = infeasibleBenchMemo(&candidateInfo.result, - srcBuffer, srcSize, - dstBuffer, dstSize, - dictBuffer, dictSize, - fileSizes, nbFiles, - ctx, dctx, - candidateInfo.params, target, &winnerInfo.result, memoTable, - varArray, varLen); - if(res == FEASIBLE_RESULT) { /* synonymous with better when called w/ infeasibleBM */ - winnerInfo = candidateInfo; - BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, srcSize); - better = 1; - if(feasible(candidateInfo.result, target) && objective_lt(bestFeasible1.result, winnerInfo.result)) { - bestFeasible1 = winnerInfo; - } - } - } - candidateInfo.params = cparam; - paramVaryOnce(varArray[i], -1, &candidateInfo.params); /* -1 */ - candidateInfo.params = sanitizeParams(candidateInfo.params); - //evaluate - if(!ZSTD_isError(ZSTD_checkCParams(candidateInfo.params))) { - //if(cParamValid(candidateInfo.params)) { - int res = infeasibleBenchMemo(&candidateInfo.result, - srcBuffer, srcSize, - dstBuffer, dstSize, - dictBuffer, dictSize, - fileSizes, nbFiles, - ctx, dctx, - candidateInfo.params, target, &winnerInfo.result, memoTable, - varArray, varLen); - if(res == FEASIBLE_RESULT) { - winnerInfo = candidateInfo; - BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, srcSize); - better = 1; - if(feasible(candidateInfo.result, target) && objective_lt(bestFeasible1.result, winnerInfo.result)) { - bestFeasible1 = winnerInfo; + for(offset = -1; offset <= 1; offset += 2) { + candidateInfo.params = cparam; + paramVaryOnce(varArray[i], offset, &candidateInfo.params); /* +1 */ + candidateInfo.params = sanitizeParams(candidateInfo.params); + //evaluate + if(!ZSTD_isError(ZSTD_checkCParams(candidateInfo.params))) { + int res = benchMemo(&candidateInfo.result, + buf, ctx, + candidateInfo.params, target, &winnerInfo.result, memoTable, + varArray, varLen, feas); + if(res == BETTER_RESULT) { /* synonymous with better when called w/ infeasibleBM */ + winnerInfo = candidateInfo; + BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, buf.srcSize); + better = 1; + if(compareResultLT(bestFeasible1.result, winnerInfo.result, target, buf.srcSize)) { + bestFeasible1 = winnerInfo; + } } } } @@ -1966,235 +1533,255 @@ static winnerInfo_t climbOnce(const constraint_t target, if(better) { continue; } - //if 'better' enough, skip further parameter search, center there? - //possible improvement - guide direction here w/ knowledge rather than completely random variation. - for(d = 2; d < varLen + 2; d++) { /* varLen is # dimensions */ + + for(dist = 2; dist < varLen + 2; dist++) { /* varLen is # dimensions */ for(i = 0; i < 2 * varLen + 2; i++) { int res; candidateInfo.params = cparam; /* param error checking already done here */ - paramVariation(&candidateInfo.params, varArray, varLen, d); - res = infeasibleBenchMemo(&candidateInfo.result, - srcBuffer, srcSize, - dstBuffer, dstSize, - dictBuffer, dictSize, - fileSizes, nbFiles, - ctx, dctx, - candidateInfo.params, target, &winnerInfo.result, memoTable, - varArray, varLen); - if(res == FEASIBLE_RESULT) { /* synonymous with better in this case*/ + paramVariation(&candidateInfo.params, varArray, varLen, dist); + res = benchMemo(&candidateInfo.result, + buf, ctx, + candidateInfo.params, target, &winnerInfo.result, memoTable, + varArray, varLen, feas); + if(res == BETTER_RESULT) { /* synonymous with better in this case*/ winnerInfo = candidateInfo; - BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, srcSize); + BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, buf.srcSize); better = 1; - if(feasible(candidateInfo.result, target) && objective_lt(bestFeasible1.result, winnerInfo.result)) { + if(compareResultLT(bestFeasible1.result, winnerInfo.result, target, buf.srcSize)) { bestFeasible1 = winnerInfo; } } } if(better) { - continue; + break; } } - //bias to test previous delta? - //change cparam -> candidate before restart + + if(!better) { //infeas -> feas -> stop. + if(feas) { return winnerInfo; } + + feas = 1; + better = 1; + winnerInfo = bestFeasible1; /* note with change, bestFeasible may not necessarily be feasible, but if one has been benchmarked, it will be. */ + DISPLAY("Climb Part 2\n"); + } } winnerInfo = bestFeasible1; } - //break out if no feasible. - if(winnerInfo.result.cMem == (U32)-1) { - DEBUGOUTPUT("No Feasible Found\n"); - return winnerInfo; - } - DISPLAY("Climb Part 2\n"); - - better = 1; - /* feasible -> best feasible (hopefully) */ - { - while(better) { - int i, d; - better = 0; - BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, srcSize); - //all dist-1 targets - cparam = winnerInfo.params; - candidateInfo.params = cparam; - for(i = 0; i < varLen; i++) { - paramVaryOnce(varArray[i], 1, &candidateInfo.params); - candidateInfo.params = sanitizeParams(candidateInfo.params); - - //evaluate - if(!ZSTD_isError(ZSTD_checkCParams(candidateInfo.params))) { - //if(cParamValid(candidateInfo.params)) { - int res = feasibleBenchMemo(&candidateInfo.result, - srcBuffer, srcSize, - dstBuffer, dstSize, - dictBuffer, dictSize, - fileSizes, nbFiles, - ctx, dctx, - candidateInfo.params, target, &winnerInfo.result, memoTable, - varArray, varLen); - if(res == FEASIBLE_RESULT) { - winnerInfo = candidateInfo; - BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, srcSize); - better = 1; - } - } - candidateInfo.params = cparam; - paramVaryOnce(varArray[i], -1, &candidateInfo.params); - candidateInfo.params = sanitizeParams(candidateInfo.params); - //evaluate - if(!ZSTD_isError(ZSTD_checkCParams(candidateInfo.params))) { - int res = feasibleBenchMemo(&candidateInfo.result, - srcBuffer, srcSize, - dstBuffer, dstSize, - dictBuffer, dictSize, - fileSizes, nbFiles, - ctx, dctx, - candidateInfo.params, target, &winnerInfo.result, memoTable, - varArray, varLen); - if(res == FEASIBLE_RESULT) { - winnerInfo = candidateInfo; - BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, srcSize); - better = 1; - } - } - } - //if 'better' enough, skip further parameter search, center there? - //possible improvement - guide direction here w/ knowledge rather than completely random variation. - for(d = 2; d < varLen + 2; d++) { /* varLen is # dimensions */ - for(i = 0; i < 2 * varLen + 2; i++) { - int res; - candidateInfo.params = cparam; - /* param error checking already done here */ - paramVariation(&candidateInfo.params, varArray, varLen, d); //info candidateInfo.params is garbage, this is too. - res = feasibleBenchMemo(&candidateInfo.result, - srcBuffer, srcSize, - dstBuffer, dstSize, - dictBuffer, dictSize, - fileSizes, nbFiles, - ctx, dctx, - candidateInfo.params, target, &winnerInfo.result, memoTable, - varArray, varLen); - if(res == FEASIBLE_RESULT) { - winnerInfo = candidateInfo; - BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, srcSize); - better = 1; - } - } - if(better) { - continue; - } - } - //bias to test previous delta? - //change cparam -> candidate before restart - } - } - return winnerInfo; } //optimizeForSize but with fixed strategy //place to configure/filter out strategy specific parameters. -//need args for all buffers and parameter stuff -//sanitization here. //flexible parameters: iterations of (failed?) climbing (or if we do non-random, maybe this is when everything is close to visitied) //weight more on visit for bad results, less on good results/more on later results / ones with more failures. //allocate memoTable here. //only real use for paramTarget is to get the fixed values, right? +//maybe allow giving it a first init? static winnerInfo_t optimizeFixedStrategy( - const void* srcBuffer, const size_t srcSize, - void* dstBuffer, const size_t dstSize, - void* dictBuffer, const size_t dictSize, - const size_t* fileSizes, const size_t nbFiles, - const constraint_t target, ZSTD_compressionParameters paramTarget, - const ZSTD_strategy strat, const U32* varArray, const int varLen, U8* memoTable) { + buffers_t buf, contexts_t ctx, + const constraint_t target, ZSTD_compressionParameters paramTarget, + const ZSTD_strategy strat, + const varInds_t* varArray, const int varLen, + U8* memoTable, const int tries) { int i = 0; - U32* varNew = malloc(sizeof(U32) * varLen); - int varLenNew = sanitizeVarArray(varLen, varArray, varNew, strat); + varInds_t varNew[NUM_PARAMS]; + int varLenNew = sanitizeVarArray(varNew, varLen, varArray, strat); ZSTD_compressionParameters init; - ZSTD_CCtx* ctx = ZSTD_createCCtx(); - ZSTD_DCtx* dctx = ZSTD_createDCtx(); winnerInfo_t winnerInfo, candidateInfo; - winnerInfo.result.cSpeed = 0; - winnerInfo.result.dSpeed = 0; - winnerInfo.result.cMem = (size_t)(-1LL); - winnerInfo.result.cSize = (size_t)(-1LL); + winnerInfo = initWinnerInfo(emptyParams()); /* so climb is given the right fixed strategy */ paramTarget.strategy = strat; /* to pass ZSTD_checkCParams */ - //needs to happen after memoTableInit as that assumes 0 = undefined. cParamZeroMin(¶mTarget); init = paramTarget; - - if(!ctx || !dctx || !memoTable || !varNew) { - DISPLAY("NOT ENOUGH MEMORY ! ! ! \n"); - goto _cleanUp; - } - - while(i < 10) { //make i adjustable (user input?) depending on how much time they have. + while(i < tries) { //make i adjustable (user input?) depending on how much time they have. DEBUGOUTPUT("Restart\n"); //look into improving this to maximize distance from searched infeasible stuff / towards promising regions? randomConstrainedParams(&init, varNew, varLenNew, memoTable); - candidateInfo = climbOnce(target, varNew, varLenNew, memoTable, srcBuffer, srcSize, dstBuffer, dstSize, dictBuffer, dictSize, fileSizes, nbFiles, ctx, dctx, init); - if(objective_lt(winnerInfo.result, candidateInfo.result)) { + candidateInfo = climbOnce(target, varNew, varLenNew, memoTable, buf, ctx, init); + if(compareResultLT(winnerInfo.result, candidateInfo.result, target, buf.srcSize)) { winnerInfo = candidateInfo; - DISPLAY("New Winner: "); - BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, srcSize); + BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, buf.srcSize); i = 0; } i++; } - -_cleanUp: - ZSTD_freeCCtx(ctx); - ZSTD_freeDCtx(dctx); - free(varNew); return winnerInfo; } -static int BMK_loadFiles(void* buffer, size_t bufferSize, - size_t* fileSizes, const char* const * const fileNamesTable, - unsigned nbFiles) +static void freeBuffers(buffers_t b) { + if(b.srcPtrs != NULL) { + free(b.srcPtrs[0]); + } + free(b.srcPtrs); + free(b.srcSizes); + + if(b.dstPtrs != NULL) { + free(b.dstPtrs[0]); + } + free(b.dstPtrs); + free(b.dstCapacities); + free(b.dstSizes); + + if(b.resPtrs != NULL) { + free(b.resPtrs[0]); + } + free(b.resPtrs); +} + +/* allocates buffer's arguments. returns success / failuere */ +static int initBuffers(buffers_t* buff, const char* const * const fileNamesTable, + size_t nbFiles) { - size_t pos = 0, totalSize = 0; - unsigned n; - for (n=0; nsrcPtrs = (void**)calloc(maxNbBlocks, sizeof(void*)); + buff->srcSizes = (size_t*)malloc(maxNbBlocks * sizeof(size_t)); + + buff->dstPtrs = (void**)calloc(maxNbBlocks, sizeof(void*)); + buff->dstCapacities = (size_t*)malloc(maxNbBlocks * sizeof(size_t)); + buff->dstSizes = (size_t*)malloc(maxNbBlocks * sizeof(size_t)); + + buff->resPtrs = (void**)calloc(maxNbBlocks, sizeof(void*)); + buff->resSizes = (size_t*)malloc(maxNbBlocks * sizeof(size_t)); + + if(!buff->srcPtrs || !buff->srcSizes || !buff->dstPtrs || !buff->dstCapacities || !buff->dstCapacities || !buff->resPtrs || !buff->resSizes) { + DISPLAY("alloc error\n"); + freeBuffers(*buff); + return 1; + } + + buff->srcPtrs[0] = malloc(benchedSize); + buff->dstPtrs[0] = malloc(ZSTD_compressBound(benchedSize) + (maxNbBlocks * 1024)); + buff->resPtrs[0] = malloc(benchedSize); + + if(!buff->srcPtrs[0] || !buff->dstPtrs[0] || !buff->resPtrs[0]) { + DISPLAY("alloc error\n"); + freeBuffers(*buff); + return 1; + } + + for(n = 0; n < nbFiles; n++) { FILE* f; U64 fileSize = UTIL_getFileSize(fileNamesTable[n]); if (UTIL_isDirectory(fileNamesTable[n])) { DISPLAY("Ignoring %s directory... \n", fileNamesTable[n]); - fileSizes[n] = 0; continue; } if (fileSize == UTIL_FILESIZE_UNKNOWN) { DISPLAY("Cannot evaluate size of %s, ignoring ... \n", fileNamesTable[n]); - fileSizes[n] = 0; continue; } f = fopen(fileNamesTable[n], "rb"); if (f==NULL) { - DISPLAY("impossible to open file %s", fileNamesTable[n]); + DISPLAY("impossible to open file %s\n", fileNamesTable[n]); + freeBuffers(*buff); + fclose(f); return 10; } + DISPLAY("Loading %s... \r", fileNamesTable[n]); - if (fileSize > bufferSize-pos) fileSize = bufferSize-pos, nbFiles=n; /* buffer too small - stop after this file */ - { size_t const readSize = fread(((char*)buffer)+pos, 1, (size_t)fileSize, f); + + if (fileSize + pos > benchedSize) fileSize = benchedSize - pos, nbFiles=n; /* buffer too small - stop after this file */ + { + char* buffer = (char*)(buff->srcPtrs[0]); + size_t const readSize = fread((buffer)+pos, 1, (size_t)fileSize, f); + size_t blocked = 0; + while(blocked < readSize) { + buff->srcPtrs[blockNb] = (buffer) + (pos + blocked); + buff->srcSizes[blockNb] = blockSize; + blocked += blockSize; + blockNb++; + } + if(readSize > 0) { buff->srcSizes[blockNb - 1] = ((readSize - 1) % blockSize) + 1; } + if (readSize != (size_t)fileSize) { DISPLAY("could not read %s", fileNamesTable[n]); - return 11; + freeBuffers(*buff); + fclose(f); + return 1; } - pos += readSize; } - fileSizes[n] = (size_t)fileSize; - totalSize += (size_t)fileSize; + + pos += readSize; + + } fclose(f); } - if (totalSize == 0) { DISPLAY("\nno data to bench\n"); return 12; } + buff->dstCapacities[0] = ZSTD_compressBound(buff->srcSizes[0]); + buff->dstSizes[0] = buff->dstCapacities[0]; + buff->resSizes[0] = buff->srcSizes[0]; + + for(n = 1; n < blockNb; n++) { + buff->dstPtrs[n] = ((char*)buff->dstPtrs[n-1]) + buff->dstCapacities[n-1]; + buff->resPtrs[n] = ((char*)buff->resPtrs[n-1]) + buff->resSizes[n-1]; + buff->dstCapacities[n] = ZSTD_compressBound(buff->srcSizes[n]); + buff->dstSizes[n] = buff->dstCapacities[n]; + buff->resSizes[n] = buff->srcSizes[n]; + } + buff->srcSize = pos; + buff->nbBlocks = blockNb; + + if (pos == 0) { DISPLAY("\nno data to bench\n"); return 1; } + + return 0; +} + +static void freeContexts(contexts_t ctx) { + free(ctx.dictBuffer); + ZSTD_freeCCtx(ctx.cctx); + ZSTD_freeDCtx(ctx.dctx); +} + +static int initContexts(contexts_t* ctx, const char* dictFileName) { + FILE* f; + size_t readSize; + ctx->cctx = ZSTD_createCCtx(); + ctx->dctx = ZSTD_createDCtx(); + if(dictFileName == NULL) { + ctx->dictSize = 0; + ctx->dictBuffer = NULL; + return 0; + } + ctx->dictSize = UTIL_getFileSize(dictFileName); + ctx->dictBuffer = malloc(ctx->dictSize); + + f = fopen(dictFileName, "rb"); + + if(!f) { + DISPLAY("unable to open file\n"); + fclose(f); + freeContexts(*ctx); + return 1; + } + + if(ctx->dictSize > 64 MB || !(ctx->dictBuffer)) { + DISPLAY("dictionary too large\n"); + fclose(f); + freeContexts(*ctx); + return 1; + } + readSize = fread(ctx->dictBuffer, 1, ctx->dictSize, f); + if(readSize != ctx->dictSize) { + DISPLAY("unable to read file\n"); + fclose(f); + freeContexts(*ctx); + return 1; + } return 0; } @@ -2228,107 +1815,98 @@ static int nextStrategy(const int currentStrategy, const int bestStrategy) { } } +static ZSTD_compressionParameters maskParams(ZSTD_compressionParameters base, ZSTD_compressionParameters mask) { + base.windowLog = mask.windowLog ? mask.windowLog : base.windowLog; + base.chainLog = mask.chainLog ? mask.chainLog : base.chainLog; + base.hashLog = mask.hashLog ? mask.hashLog : base.hashLog; + base.searchLog = mask.searchLog ? mask.searchLog : base.searchLog; + base.searchLength = mask.searchLength ? mask.searchLength : base.searchLength; + base.targetLength = mask.targetLength ? mask.targetLength : base.targetLength; + base.strategy = mask.strategy ? mask.strategy : base.strategy; + return base; +} + +#define MAX_TRIES 8 //optimize fixed strategy. static int optimizeForSize(const char* const * const fileNamesTable, const size_t nbFiles, const char* dictFileName, constraint_t target, ZSTD_compressionParameters paramTarget, int cLevel) { - size_t benchedSize; - void* origBuff = NULL; - void* dictBuffer = NULL; - size_t dictBufferSize = 0; - U32 varArray [NUM_PARAMS]; + varInds_t varArray [NUM_PARAMS]; int ret = 0; - size_t* fileSizes = calloc(sizeof(size_t),nbFiles); const int varLen = variableParams(paramTarget, varArray); + winnerInfo_t winner = initWinnerInfo(emptyParams()); U8** allMT = NULL; - g_winner.result.cSize = (size_t)-1; + size_t k; + size_t maxBlockSize = 0; + contexts_t ctx; + buffers_t buf; + /* Init */ if(!cParamValid(paramTarget)) { - return 10; + return 1; } /* load dictionary*/ - if (dictFileName != NULL) { - U64 const dictFileSize = UTIL_getFileSize(dictFileName); - if (dictFileSize > 64 MB) { - DISPLAY("dictionary file %s too large", dictFileName); - ret = 10; - goto _cleanUp; - } - dictBufferSize = (size_t)dictFileSize; - dictBuffer = malloc(dictBufferSize); - if (dictBuffer==NULL) { - DISPLAY("not enough memory for dictionary (%u bytes)", - (U32)dictBufferSize); - ret = 11; - goto _cleanUp; + if(initBuffers(&buf, fileNamesTable, nbFiles)) { + DISPLAY("unable to load files\n"); + return 1; + } - } - - { - int errorCode = BMK_loadFiles(dictBuffer, dictBufferSize, &dictBufferSize, &dictFileName, 1); - if(errorCode) { - ret = errorCode; - goto _cleanUp; - } - } + if(initContexts(&ctx, dictFileName)) { + DISPLAY("unable to load dictionary\n"); + freeBuffers(buf); + return 2; } - /* Fill input buffer */ if(nbFiles == 1) { DISPLAY("Loading %s... \r", fileNamesTable[0]); } else { DISPLAY("Loading %lu Files... \r", (unsigned long)nbFiles); } - { - U64 const totalSizeToLoad = UTIL_getTotalFileSize(fileNamesTable, (U32)nbFiles); - int ec; - unsigned i; - benchedSize = BMK_findMaxMem(totalSizeToLoad * 3) / 3; - origBuff = malloc(benchedSize); - if(!origBuff || !fileSizes) { - DISPLAY("Not enough memory for stuff\n"); - ret = 1; - goto _cleanUp; - } - ec = BMK_loadFiles(origBuff, benchedSize, fileSizes, fileNamesTable, (U32)nbFiles); - if(ec) { - DISPLAY("Error Loading Files"); - ret = ec; - goto _cleanUp; - } - benchedSize = 0; - for(i = 0; i < nbFiles; i++) { - benchedSize += fileSizes[i]; - } - origBuff = realloc(origBuff, benchedSize); + for(k = 0; k < buf.nbBlocks; k++) { + maxBlockSize = MAX(buf.srcSizes[k], maxBlockSize); } - allMT = memoTableInitAll(paramTarget, target, varArray, varLen, benchedSize); + /* if strategy is fixed, only init that part of memotable */ + if(paramTarget.strategy) { + varInds_t varNew[NUM_PARAMS]; + int varLenNew = sanitizeVarArray(varNew, varLen, varArray, paramTarget.strategy); + allMT = calloc(sizeof(U8), (ZSTD_btultra + 1)); + if(allMT == NULL) { + ret = 57; + goto _cleanUp; + } + + allMT[paramTarget.strategy] = malloc(sizeof(U8) * memoTableLen(varNew, varLenNew)); + + if(allMT[paramTarget.strategy] == NULL) { + ret = 58; + goto _cleanUp; + } + + memoTableInit(allMT[paramTarget.strategy], paramTarget, target, varNew, varLenNew, maxBlockSize); + } else { + allMT = memoTableInitAll(paramTarget, target, varArray, varLen, maxBlockSize); + } + + if(!allMT) { + DISPLAY("MemoTable Init Error\n"); ret = 2; goto _cleanUp; } if(cLevel) { - BMK_result_t candidate; - const size_t blockSize = g_blockSize ? g_blockSize : benchedSize; - ZSTD_CCtx* const ctx = ZSTD_createCCtx(); - ZSTD_DCtx* const dctx = ZSTD_createDCtx(); - ZSTD_compressionParameters const CParams = ZSTD_getCParams(cLevel, blockSize, dictBufferSize); - if(BMK_benchParam(&candidate, origBuff, benchedSize, fileSizes, (U32)nbFiles, ctx, dctx, CParams)) { - ZSTD_freeCCtx(ctx); - ZSTD_freeDCtx(dctx); + winner.params = ZSTD_getCParams(cLevel, maxBlockSize, ctx.dictSize); + if(BMK_benchParam(&winner.result, buf, ctx, winner.params)) { ret = 3; goto _cleanUp; } - target.cSpeed = (U32)candidate.cSpeed; //Maybe have a small bit of slack here, like x.99? - BMK_printWinner(stdout, cLevel, candidate, CParams, benchedSize); - - ZSTD_freeCCtx(ctx); - ZSTD_freeDCtx(dctx); + target.cSpeed = (U32)winner.result.cSpeed; //Maybe have a small bit of slack here, like x.99? + g_targetConstraints = target; + BMK_printWinner(stdout, cLevel, winner.result, winner.params, buf.srcSize); } g_targetConstraints = target; @@ -2340,106 +1918,40 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ } else { DISPLAY("optimizing for %lu Files", (unsigned long)nbFiles); } - if(target.cSpeed != 0) { DISPLAY(" - limit compression speed %u MB/s", target.cSpeed / 1000000); } - if(target.dSpeed != 0) { DISPLAY(" - limit decompression speed %u MB/s", target.dSpeed / 1000000); } - if(target.cMem != (U32)-1) { DISPLAY(" - limit memory %u MB", target.cMem / 1000000); } + if(target.cSpeed != 0) { DISPLAY(" - limit compression speed %u MB/s", target.cSpeed >> 20); } + if(target.dSpeed != 0) { DISPLAY(" - limit decompression speed %u MB/s", target.dSpeed >> 20); } + if(target.cMem != (U32)-1) { DISPLAY(" - limit memory %u MB", target.cMem >> 20); } + DISPLAY("\n"); findClockGranularity(); - { ZSTD_CCtx* const ctx = ZSTD_createCCtx(); - ZSTD_DCtx* const dctx = ZSTD_createDCtx(); - winnerInfo_t winner; - U32 varNew[NUM_PARAMS]; - const size_t blockSize = g_blockSize ? g_blockSize : benchedSize; - U32 const maxNbBlocks = (U32) ((benchedSize + (blockSize-1)) / blockSize) + 1; - const size_t maxCompressedSize = ZSTD_compressBound(benchedSize) + (maxNbBlocks * 1024); - void* compressedBuffer = malloc(maxCompressedSize); - - /* init */ - if (ctx==NULL) { DISPLAY("\n ZSTD_createCCtx error \n"); free(origBuff); return 14;} - if(compressedBuffer==NULL) { DISPLAY("\n Allocation Error \n"); free(origBuff); free(ctx); return 15; } - memset(&winner, 0, sizeof(winner)); - winner.result.cSize = (size_t)(-1); - + { + varInds_t varNew[NUM_PARAMS]; /* find best solution from default params */ { /* strategy selection */ const int maxSeeds = g_noSeed ? 1 : ZSTD_maxCLevel(); DEBUGOUTPUT("Strategy Selection\n"); - if(varLen == NUM_PARAMS && paramTarget.strategy == 0) { /* no variable based constraints */ + if(paramTarget.strategy == 0) { /* no variable based constraints */ BMK_result_t candidate; - int feas = 0, i; + int i; for (i=1; i<=maxSeeds; i++) { - ZSTD_compressionParameters const CParams = ZSTD_getCParams(i, blockSize, dictBufferSize); - int ec = BMK_benchParam(&candidate, origBuff, benchedSize, fileSizes, (U32)nbFiles, ctx, dctx, CParams); - BMK_printWinner(stdout, i, candidate, CParams, benchedSize); - - if(!ec) { - if(feas) { - if(feasible(candidate, relaxTarget(target)) && objective_lt(winner.result, candidate)) { - winner.result = candidate; - winner.params = CParams; - } - } else { - if(feasible(candidate, relaxTarget(target))) { - feas = 1; - winner.result = candidate; - winner.params = CParams; - - } else { - if(resultScore(candidate, benchedSize, target) > resultScore(winner.result, benchedSize, target)) { - winner.result = candidate; - winner.params = CParams; - } - } - } - } - } //best, -1, +1, ..., - - } else if (paramTarget.strategy == 0) { //constrained - int feas = 0, i, j; - for(j = 1; j < 10; j++) { - for(i = 1; i <= maxSeeds; i++) { - int varLenNew = sanitizeVarArray(varLen, varArray, varNew, i); - ZSTD_compressionParameters candidateParams = paramTarget; - BMK_result_t candidate; - int ec; - randomConstrainedParams(&candidateParams, varNew, varLenNew, allMT[i]); - cParamZeroMin(&candidateParams); - candidateParams = sanitizeParams(candidateParams); - ec = BMK_benchParam(&candidate, origBuff, benchedSize, fileSizes, (U32)nbFiles, ctx, dctx, candidateParams); - - if(!ec) { - if(feas) { - if(feasible(candidate, relaxTarget(target)) && objective_lt(winner.result, candidate)) { - winner.result = candidate; - winner.params = candidateParams; - BMK_printWinner(stdout, i, winner.result, winner.params, benchedSize); - } - } else { - if(feasible(candidate, relaxTarget(target))) { - feas = 1; - winner.result = candidate; - winner.params = candidateParams; - BMK_printWinner(stdout, i, winner.result, winner.params, benchedSize); - - } else { - if(resultScore(candidate, benchedSize, target) > resultScore(winner.result, benchedSize, target)) { - winner.result = candidate; - winner.params = candidateParams; - BMK_printWinner(stdout, i, winner.result, winner.params, benchedSize); - } - } - } - } + int ec; + ZSTD_compressionParameters CParams = ZSTD_getCParams(i, maxBlockSize, ctx.dictSize); + CParams = maskParams(CParams, paramTarget); + ec = BMK_benchParam(&candidate, buf, ctx, CParams); + BMK_printWinner(stdout, i, candidate, CParams, buf.srcSize); + if(!ec && compareResultLT(winner.result, candidate, relaxTarget(target), buf.srcSize)) { + winner.result = candidate; + winner.params = CParams; } } } } - BMK_printWinner(stdout, CUSTOM_LEVEL, winner.result, winner.params, benchedSize); + BMK_printWinner(stdout, CUSTOM_LEVEL, winner.result, winner.params, buf.srcSize); BMK_translateAdvancedParams(winner.params); DEBUGOUTPUT("Real Opt\n"); /* start 'real' tests */ @@ -2447,55 +1959,51 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ int bestStrategy = (int)winner.params.strategy; if(paramTarget.strategy == 0) { int st = (int)winner.params.strategy; + int tries = MAX_TRIES; { - int varLenNew = sanitizeVarArray(varLen, varArray, varNew, st); + int varLenNew = sanitizeVarArray(varNew, varLen, varArray, st); winnerInfo_t w1 = climbOnce(target, varNew, varLenNew, allMT[st], - origBuff, benchedSize, compressedBuffer, maxCompressedSize, dictBuffer, dictBufferSize, - fileSizes, nbFiles, ctx, dctx, winner.params); - if(objective_lt(winner.result, w1.result)) { + buf, ctx, winner.params); + if(compareResultLT(winner.result, w1.result, target, buf.srcSize)) { winner = w1; } } - while(st) { - winnerInfo_t wc = optimizeFixedStrategy(origBuff, benchedSize, compressedBuffer, maxCompressedSize, dictBuffer, dictBufferSize, fileSizes, nbFiles, - target, paramTarget, st, varArray, varLen, allMT[st]); + while(st && tries) { + winnerInfo_t wc = optimizeFixedStrategy(buf, ctx, target, paramTarget, + st, varArray, varLen, allMT[st], tries); DEBUGOUTPUT("StratNum %d\n", st); - if(objective_lt(winner.result, wc.result)) { + if(compareResultLT(winner.result, wc.result, target, buf.srcSize)) { winner = wc; } //We could double back to increase search of 'better' strategies st = nextStrategy(st, bestStrategy); + tries--; } } else { - winner = optimizeFixedStrategy(origBuff, benchedSize, compressedBuffer, maxCompressedSize, dictBuffer, dictBufferSize, fileSizes, nbFiles, - target, paramTarget, paramTarget.strategy, varArray, varLen, allMT[paramTarget.strategy]); + winner = optimizeFixedStrategy(buf, ctx, target, paramTarget, paramTarget.strategy, + varArray, varLen, allMT[paramTarget.strategy], 10); } } /* no solution found */ if(winner.result.cSize == (size_t)-1) { + ret = 1; DISPLAY("No feasible solution found\n"); - return 1; + goto _cleanUp; } /* end summary */ - BMK_printWinner(stdout, CUSTOM_LEVEL, winner.result, winner.params, benchedSize); + BMK_printWinner(stdout, CUSTOM_LEVEL, winner.result, winner.params, buf.srcSize); BMK_translateAdvancedParams(winner.params); DISPLAY("grillParams size - optimizer completed \n"); - - /* clean up*/ - ZSTD_freeCCtx(ctx); - ZSTD_freeDCtx(dctx); - } _cleanUp: - free(fileSizes); - free(dictBuffer); + freeContexts(ctx); + freeBuffers(buf); memoTableFreeAll(allMT); - free(origBuff); return ret; } @@ -2546,16 +2054,16 @@ static int usage(const char* exename) static int usage_advanced(void) { DISPLAY( "\nAdvanced options :\n"); - DISPLAY( " -T# : set level 1 speed objective \n"); - DISPLAY( " -B# : cut input into blocks of size # (default : single block) \n"); - DISPLAY( " -i# : iteration loops [1-9](default : %i) \n", NBLOOPS); - DISPLAY( " -O# : find Optimized parameters for # MB/s compression speed (default : 0) \n"); - DISPLAY( " -S : Single run \n"); - DISPLAY( " --zstd : Single run, parameter selection same as zstdcli \n"); - DISPLAY( " -P# : generated sample compressibility (default : %.1f%%) \n", COMPRESSIBILITY_DEFAULT * 100); - DISPLAY( " -t# : Caps runtime of operation in seconds (default : %u seconds (%.1f hours)) \n", (U32)g_grillDuration_s, g_grillDuration_s / 3600); - DISPLAY( " -v : Prints Benchmarking output\n"); - DISPLAY( " -D : Next argument dictionary file\n"); + DISPLAY( " -T# : set level 1 speed objective \n"); + DISPLAY( " -B# : cut input into blocks of size # (default : single block) \n"); + DISPLAY( " -i# : iteration loops (default : %i) \n", NBLOOPS); + DISPLAY( " --optimize= : same as -O with more verbose syntax (see README.md)\n"); + DISPLAY( " -S : Single run \n"); + DISPLAY( " --zstd : Single run, parameter selection same as zstdcli \n"); + DISPLAY( " -P# : generated sample compressibility (default : %.1f%%) \n", COMPRESSIBILITY_DEFAULT * 100); + DISPLAY( " -t# : Caps runtime of operation in seconds (default : %u seconds (%.1f hours)) \n", (U32)g_grillDuration_s, g_grillDuration_s / 3600); + DISPLAY( " -v : Prints Benchmarking output\n"); + DISPLAY( " -D : Next argument dictionary file\n"); return 0; } @@ -2572,8 +2080,8 @@ int main(int argc, const char** argv) filenamesStart=0, result; const char* exename=argv[0]; - const char* input_filename = 0; - const char* dictFileName = 0; + const char* input_filename = NULL; + const char* dictFileName = NULL; U32 optimizer = 0; U32 main_pause = 0; int optimizerCLevel = 0; @@ -2591,6 +2099,9 @@ int main(int argc, const char** argv) for(i=1; i Date: Fri, 27 Jul 2018 08:49:25 -0700 Subject: [PATCH 119/372] Renaming / Style fixes --- programs/bench.c | 56 +++++++++++++++++++++++++--------------------- programs/bench.h | 7 +++--- tests/README.md | 1 - tests/paramgrill.c | 27 +++++++++++----------- 4 files changed, 48 insertions(+), 43 deletions(-) diff --git a/programs/bench.c b/programs/bench.c index 177dbe0e3..b496caf27 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -294,7 +294,7 @@ BMK_customReturn_t BMK_benchFunction( BMK_initFn_t initFn, void* initPayload, size_t blockCount, const void* const * const srcBlockBuffers, const size_t* srcBlockSizes, - void* const * const dstBlockBuffers, const size_t* dstBlockCapacities, size_t* cSizes, + void* const * const dstBlockBuffers, const size_t* dstBlockCapacities, size_t* blockResult, unsigned nbLoops) { size_t dstSize = 0; U64 totalTime; @@ -332,8 +332,8 @@ BMK_customReturn_t BMK_benchFunction( j, (U32)dstBlockCapacities[j], ZSTD_getErrorName(res)); } else if(i == nbLoops - 1) { dstSize += res; - if(cSizes != NULL) { - cSizes[j] = res; + if(blockResult != NULL) { + blockResult[j] = res; } } } @@ -358,6 +358,9 @@ void BMK_resetTimeState(BMK_timedFnState_t* r, unsigned nbSeconds) { BMK_timedFnState_t* BMK_createTimeState(unsigned nbSeconds) { BMK_timedFnState_t* r = (BMK_timedFnState_t*)malloc(sizeof(struct BMK_timeState_t)); + if(r == NULL) { + return r; + } BMK_resetTimeState(r, nbSeconds); return r; } @@ -373,7 +376,7 @@ BMK_customTimedReturn_t BMK_benchFunctionTimed( BMK_initFn_t initFn, void* initPayload, size_t blockCount, const void* const* const srcBlockBuffers, const size_t* srcBlockSizes, - void * const * const dstBlockBuffers, const size_t * dstBlockCapacities, size_t* dstSizes) + void * const * const dstBlockBuffers, const size_t * dstBlockCapacities, size_t* blockResults) { U64 fastest = cont->fastestTime; int completed = 0; @@ -390,7 +393,7 @@ BMK_customTimedReturn_t BMK_benchFunctionTimed( } /* reinitialize capacity */ r.result = BMK_benchFunction(benchFn, benchPayload, initFn, initPayload, - blockCount, srcBlockBuffers, srcBlockSizes, dstBlockBuffers, dstBlockCapacities, dstSizes, cont->nbLoops); + blockCount, srcBlockBuffers, srcBlockSizes, dstBlockBuffers, dstBlockCapacities, blockResults, cont->nbLoops); if(r.result.error) { /* completed w/ error */ r.completed = 1; return r; @@ -718,38 +721,36 @@ BMK_return_t BMK_benchMemAdvanced(const void* srcBuffer, size_t srcSize, U32 const maxNbBlocks = (U32) ((srcSize + (blockSize-1)) / blockSize) + nbFiles; /* these are the blockTable parameters, just split up */ - const void ** const srcPtrs = (const void** const)malloc(maxNbBlocks * sizeof(void*)); - size_t* const srcSizes = (size_t* const)malloc(maxNbBlocks * sizeof(size_t)); + const void ** const srcPtrs = (const void**)malloc(maxNbBlocks * sizeof(void*)); + size_t* const srcSizes = (size_t*)malloc(maxNbBlocks * sizeof(size_t)); - void ** const cPtrs = (void** const)malloc(maxNbBlocks * sizeof(void*)); - size_t* const cSizes = (size_t* const)malloc(maxNbBlocks * sizeof(size_t)); - size_t* const cCapacities = (size_t* const)malloc(maxNbBlocks * sizeof(size_t)); + void ** const cPtrs = (void**)malloc(maxNbBlocks * sizeof(void*)); + size_t* const cSizes = (size_t*)malloc(maxNbBlocks * sizeof(size_t)); + size_t* const cCapacities = (size_t*)malloc(maxNbBlocks * sizeof(size_t)); - void ** const resPtrs = (void** const)malloc(maxNbBlocks * sizeof(void*)); - size_t* const resSizes = (size_t* const)malloc(maxNbBlocks * sizeof(size_t)); + void ** const resPtrs = (void**)malloc(maxNbBlocks * sizeof(void*)); + size_t* const resSizes = (size_t*)malloc(maxNbBlocks * sizeof(size_t)); BMK_timedFnState_t* timeStateCompress = BMK_createTimeState(adv->nbSeconds); BMK_timedFnState_t* timeStateDecompress = BMK_createTimeState(adv->nbSeconds); - void* compressedBuffer; const size_t maxCompressedSize = dstCapacity ? dstCapacity : ZSTD_compressBound(srcSize) + (maxNbBlocks * 1024); + + void* const internalDstBuffer = dstBuffer ? NULL : malloc(maxCompressedSize); + void* const compressedBuffer = dstBuffer ? dstBuffer : internalDstBuffer; + void* resultBuffer = malloc(srcSize); - BMK_return_t results = { { 0, 0, 0, 0 }, 0 }; - int allocationincomplete; + int allocationincomplete = !srcPtrs || !srcSizes || !cPtrs || + !cSizes || !cCapacities || !resPtrs || !resSizes || + !timeStateCompress || !timeStateDecompress || !compressedBuffer || !resultBuffer; - if(!dstCapacity) { - compressedBuffer = malloc(maxCompressedSize); - } else { - compressedBuffer = dstBuffer; - } + int parametersConflict = !dstBuffer ^ !dstCapacity; - allocationincomplete = !compressedBuffer || !resultBuffer || - !srcPtrs || !srcSizes || !cPtrs || !cSizes || !resPtrs || !resSizes; - if (!allocationincomplete) { + if (!allocationincomplete && !parametersConflict) { results = BMK_benchMemAdvancedNoAlloc(srcPtrs, srcSizes, cPtrs, cCapacities, cSizes, resPtrs, resSizes, &resultBuffer, compressedBuffer, maxCompressedSize, timeStateCompress, timeStateDecompress, srcBuffer, srcSize, fileSizes, nbFiles, cLevel, comprParams, @@ -759,9 +760,8 @@ BMK_return_t BMK_benchMemAdvanced(const void* srcBuffer, size_t srcSize, /* clean up */ BMK_freeTimeState(timeStateCompress); BMK_freeTimeState(timeStateDecompress); - if(!dstCapacity) { /* only free if not given */ - free(compressedBuffer); - } + + free(internalDstBuffer); free(resultBuffer); free((void*)srcPtrs); @@ -775,6 +775,10 @@ BMK_return_t BMK_benchMemAdvanced(const void* srcBuffer, size_t srcSize, if(allocationincomplete) { EXM_THROW(31, BMK_return_t, "allocation error : not enough memory"); } + + if(parametersConflict) { + EXM_THROW(32, BMK_return_t, "Conflicting input results"); + } return results; } diff --git a/programs/bench.h b/programs/bench.h index 2a9945ac8..8baf33a0a 100644 --- a/programs/bench.h +++ b/programs/bench.h @@ -175,7 +175,8 @@ typedef size_t (*BMK_initFn_t)(void*); * srcBuffers - an array of buffers to be operated on by benchFn * srcSizes - an array of the sizes of above buffers * dstBuffers - an array of buffers to be written into by benchFn - * dstCapacitiesToSizes - an array of the capacities of above buffers. Output modified to compressed sizes of those blocks. + * dstCapacities - an array of the capacities of above buffers + * blockResults - the return value of benchFn called on each block. * nbLoops - defines number of times benchFn is run. * assumed array of size blockCount, will have compressed size of each block written to it. * return @@ -192,7 +193,7 @@ BMK_customReturn_t BMK_benchFunction(BMK_benchFn_t benchFn, void* benchPayload, BMK_initFn_t initFn, void* initPayload, size_t blockCount, const void* const * const srcBuffers, const size_t* srcSizes, - void * const * const dstBuffers, const size_t* dstCapacities, size_t* cSizes, + void * const * const dstBuffers, const size_t* dstCapacities, size_t* blockResults, unsigned nbLoops); @@ -221,7 +222,7 @@ BMK_customTimedReturn_t BMK_benchFunctionTimed(BMK_timedFnState_t* cont, BMK_initFn_t initFn, void* initPayload, size_t blockCount, const void* const * const srcBlockBuffers, const size_t* srcBlockSizes, - void* const * const dstBlockBuffers, const size_t* dstBlockCapacities, size_t* cSizes); + void* const * const dstBlockBuffers, const size_t* dstBlockCapacities, size_t* blockResults); #endif /* BENCH_H_121279284357 */ diff --git a/tests/README.md b/tests/README.md index 8bedd0a3c..2f0026fda 100644 --- a/tests/README.md +++ b/tests/README.md @@ -112,7 +112,6 @@ Full list of arguments dSpeed= - Minimum decompression speed cMem= - compression memory lvl= - Automatically sets compression speed constraint to the speed of that level - --optimize= : same as -O with more verbose syntax -P# : generated sample compressibility -t# : Caps runtime of operation in seconds (default : 99999 seconds (about 27 hours )) -v : Prints Benchmarking output diff --git a/tests/paramgrill.c b/tests/paramgrill.c index 099be3688..a4f4a5f15 100644 --- a/tests/paramgrill.c +++ b/tests/paramgrill.c @@ -986,7 +986,7 @@ static void memoTableIndInv(ZSTD_compressionParameters* ptr, const varInds_t* va } /* Initialize memotable, immediately mark redundant / obviously infeasible params as */ -static void memoTableInit(U8* memoTable, ZSTD_compressionParameters paramConstraints, const constraint_t target, const varInds_t* varyParams, const int varyLen, const size_t srcSize) { +static void createMemoTable(U8* memoTable, ZSTD_compressionParameters paramConstraints, const constraint_t target, const varInds_t* varyParams, const int varyLen, const size_t srcSize) { size_t i; size_t arrayLen = memoTableLen(varyParams, varyLen); int cwFixed = !paramConstraints.chainLog || !paramConstraints.windowLog; @@ -1053,23 +1053,24 @@ static void memoTableFreeAll(U8** mtAll) { /* inits memotables for all (including mallocs), all strategies */ /* takes unsanitized varyParams */ -static U8** memoTableInitAll(ZSTD_compressionParameters paramConstraints, constraint_t target, const varInds_t* varyParams, const int varyLen, const size_t srcSize) { +static U8** createMemoTableArray(ZSTD_compressionParameters paramConstraints, constraint_t target, const varInds_t* varyParams, const int varyLen, const size_t srcSize) { varInds_t varNew[NUM_PARAMS]; - int varLenNew; U8** mtAll = calloc(sizeof(U8*),(ZSTD_btultra + 1)); int i; if(mtAll == NULL) { return NULL; } + for(i = 1; i <= (int)ZSTD_btultra; i++) { - varLenNew = sanitizeVarArray(varNew, varyLen, varyParams, i); + const int varLenNew = sanitizeVarArray(varNew, varyLen, varyParams, i); mtAll[i] = malloc(sizeof(U8) * memoTableLen(varNew, varLenNew)); if(mtAll[i] == NULL) { memoTableFreeAll(mtAll); return NULL; } - memoTableInit(mtAll[i], paramConstraints, target, varNew, varLenNew, srcSize); + createMemoTable(mtAll[i], paramConstraints, target, varNew, varLenNew, srcSize); } + return mtAll; } @@ -1187,7 +1188,7 @@ static void BMK_benchFullTable(ZSTD_CCtx* cctx, ZSTD_DCtx* dctx, const void* src winnerInfo_t winners[NB_LEVELS_TRACKED+1]; const char* const rfName = "grillResults.txt"; FILE* const f = fopen(rfName, "w"); - const size_t blockSize = g_blockSize ? g_blockSize : ZSTD_BLOCKSIZE_MAX; /* cut by block or not ? */ + const size_t blockSize = g_blockSize ? g_blockSize : srcSize; /* cut by block or not ? */ /* init */ assert(g_singleRun==0); @@ -1638,7 +1639,7 @@ static void freeBuffers(buffers_t b) { } /* allocates buffer's arguments. returns success / failuere */ -static int initBuffers(buffers_t* buff, const char* const * const fileNamesTable, +static int createBuffers(buffers_t* buff, const char* const * const fileNamesTable, size_t nbFiles) { size_t pos = 0; @@ -1659,7 +1660,7 @@ static int initBuffers(buffers_t* buff, const char* const * const fileNamesTable buff->resPtrs = (void**)calloc(maxNbBlocks, sizeof(void*)); buff->resSizes = (size_t*)malloc(maxNbBlocks * sizeof(size_t)); - if(!buff->srcPtrs || !buff->srcSizes || !buff->dstPtrs || !buff->dstCapacities || !buff->dstCapacities || !buff->resPtrs || !buff->resSizes) { + if(!buff->srcPtrs || !buff->srcSizes || !buff->dstPtrs || !buff->dstCapacities || !buff->dstSizes || !buff->resPtrs || !buff->resSizes) { DISPLAY("alloc error\n"); freeBuffers(*buff); return 1; @@ -1747,7 +1748,7 @@ static void freeContexts(contexts_t ctx) { ZSTD_freeDCtx(ctx.dctx); } -static int initContexts(contexts_t* ctx, const char* dictFileName) { +static int createContexts(contexts_t* ctx, const char* dictFileName) { FILE* f; size_t readSize; ctx->cctx = ZSTD_createCCtx(); @@ -1846,12 +1847,12 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ } /* load dictionary*/ - if(initBuffers(&buf, fileNamesTable, nbFiles)) { + if(createBuffers(&buf, fileNamesTable, nbFiles)) { DISPLAY("unable to load files\n"); return 1; } - if(initContexts(&ctx, dictFileName)) { + if(createContexts(&ctx, dictFileName)) { DISPLAY("unable to load dictionary\n"); freeBuffers(buf); return 2; @@ -1885,9 +1886,9 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ goto _cleanUp; } - memoTableInit(allMT[paramTarget.strategy], paramTarget, target, varNew, varLenNew, maxBlockSize); + createMemoTable(allMT[paramTarget.strategy], paramTarget, target, varNew, varLenNew, maxBlockSize); } else { - allMT = memoTableInitAll(paramTarget, target, varArray, varLen, maxBlockSize); + allMT = createMemoTableArray(paramTarget, target, varArray, varLen, maxBlockSize); } From 8faeb41679a5ef379cfb7893825e6ed98e571eb0 Mon Sep 17 00:00:00 2001 From: George Lu Date: Mon, 30 Jul 2018 11:30:38 -0700 Subject: [PATCH 120/372] Update Documentation Change comment // to /* */ Add more description of what functions do Remove outdated comments --- tests/paramgrill.c | 117 ++++++++++++++++++++++++--------------------- 1 file changed, 63 insertions(+), 54 deletions(-) diff --git a/tests/paramgrill.c b/tests/paramgrill.c index a4f4a5f15..b8396c72e 100644 --- a/tests/paramgrill.c +++ b/tests/paramgrill.c @@ -77,12 +77,11 @@ typedef enum { } varInds_t; #define NUM_PARAMS 6 -//just don't use strategy as a param. +/* just don't use strategy as a param. */ #undef ZSTD_WINDOWLOG_MAX #define ZSTD_WINDOWLOG_MAX 27 //no long range stuff for now. -//make 2^[0,10] w/ 999 #define ZSTD_TARGETLENGTH_MIN 0 #define ZSTD_TARGETLENGTH_MAX 999 @@ -274,7 +273,6 @@ static double resultScore(const BMK_result_t res, const size_t srcSize, const co ret = (MIN(1, cs) + MIN(1, ds) + MIN(1, cm))*r1 + rt * rtr + (MAX(0, log(cs))+ MAX(0, log(ds))+ MAX(0, log(cm))) * r2; - //DISPLAY("resultScore: %f\n", ret); return ret; } @@ -593,7 +591,7 @@ static BMK_return_t BMK_benchMemInvertible(buffers_t buf, contexts_t ctx, BMK_freeTimeState(timeStateCompress); BMK_freeTimeState(timeStateDecompress); - } else { //iterMode; + } else { /* iterMode; */ if(mode != BMK_decodeOnly) { BMK_customReturn_t compressionResults = BMK_benchFunction(&local_defaultCompress, (void*)cctx, &local_initCCtx, (void*)&cctxprep, @@ -636,8 +634,7 @@ static BMK_return_t BMK_benchMemInvertible(buffers_t buf, contexts_t ctx, return results; } -/* global winner used for display. */ -//Should be totally 0 initialized? +/* global winner used for display. */ static winnerInfo_t g_winner = { { 0, 0, (size_t)-1, (size_t)-1 } , { 0, 0, 0, 0, 0, 0, ZSTD_fast } }; static constraint_t g_targetConstraints; @@ -940,7 +937,7 @@ static void paramVariation(ZSTD_compressionParameters* ptr, const varInds_t* var } validated = !ZSTD_isError(ZSTD_checkCParams(p)); } - *ptr = p;//sanitizeParams(p); + *ptr = p; } /* length of memo table given free variables */ @@ -953,7 +950,7 @@ static size_t memoTableLen(const varInds_t* varyParams, const int varyLen) { return arrayLen; } -/* returns unique index of compression parameters */ +/* returns unique index in memotable of compression parameters */ static unsigned memoTableInd(const ZSTD_compressionParameters* ptr, const varInds_t* varyParams, const int varyLen) { int i; unsigned ind = 0; @@ -985,7 +982,7 @@ static void memoTableIndInv(ZSTD_compressionParameters* ptr, const varInds_t* va } } -/* Initialize memotable, immediately mark redundant / obviously infeasible params as */ +/* Initialize memotable, immediately mark redundant / obviously infeasible params as such */ static void createMemoTable(U8* memoTable, ZSTD_compressionParameters paramConstraints, const constraint_t target, const varInds_t* varyParams, const int varyLen, const size_t srcSize) { size_t i; size_t arrayLen = memoTableLen(varyParams, varyLen); @@ -1003,7 +1000,7 @@ static void createMemoTable(U8* memoTable, ZSTD_compressionParameters paramConst memoTable[i] = 255; j++; } - if(wFixed && (1ULL << paramConstraints.windowLog) > (srcSize << 2)) { + if(wFixed && (1ULL << paramConstraints.windowLog) > srcSize) { memoTable[i] = 255; } /* nil out parameter sets equivalent to others. */ @@ -1121,7 +1118,7 @@ static void playAround(FILE* f, winnerInfo_t* winners, } - +/* Completely random parameter selection */ static ZSTD_compressionParameters randomParams(void) { ZSTD_compressionParameters p; @@ -1136,7 +1133,6 @@ static ZSTD_compressionParameters randomParams(void) p.targetLength=(FUZ_rand(&g_rand) % (512)); p.strategy = (ZSTD_strategy) (FUZ_rand(&g_rand) % (ZSTD_btultra +1)); validated = !ZSTD_isError(ZSTD_checkCParams(p)); - //validated = cParamValid(p); } return p; } @@ -1144,7 +1140,7 @@ static ZSTD_compressionParameters randomParams(void) /* Sets pc to random unmeasured set of parameters */ static void randomConstrainedParams(ZSTD_compressionParameters* pc, varInds_t* varArray, int varLen, U8* memoTable) { - size_t tries = memoTableLen(varArray, varLen); //configurable, + size_t tries = memoTableLen(varArray, varLen); const size_t maxSize = memoTableLen(varArray, varLen); size_t ind; do { @@ -1153,7 +1149,6 @@ static void randomConstrainedParams(ZSTD_compressionParameters* pc, varInds_t* v } while(memoTable[ind] > 0 && tries > 0); memoTableIndInv(pc, varArray, varLen, (unsigned)ind); - //*pc = sanitizeParams(*pc); } static void BMK_selectRandomStart( @@ -1328,19 +1323,13 @@ int benchFiles(const char** fileNamesTable, int nbFiles) return 0; } -/*benchmarks and tests feasibility together - 1 = true = better - 0 = false = not better - if true then resultPtr will give results. - 2+ on error? */ -//Maybe use compress_only for benchmark first run? + #define WORSE_RESULT 0 #define BETTER_RESULT 1 #define ERROR_RESULT 2 -//add worse result complete for worse results of length > 1 sec? -/* variation between 2nd run and full second bmk */ +/* Benchmarking which stops when we are sufficiently sure the solution is infeasible / worse than the winner */ #define VARIANCE 1.1 static int allBench(BMK_result_t* resultPtr, buffers_t buf, contexts_t ctx, @@ -1447,8 +1436,9 @@ static int allBench(BMK_result_t* resultPtr, } -/* wrap feasibleBench w/ memotable */ #define INFEASIBLE_THRESHOLD 200 + +/* Memoized benchmarking, won't benchmark anything which has already been benchmarked before. */ static int benchMemo(BMK_result_t* resultPtr, buffers_t buf, contexts_t ctx, const ZSTD_compressionParameters cParams, @@ -1475,18 +1465,28 @@ static int benchMemo(BMK_result_t* resultPtr, return res; } - -//sanitize all params here. -//all generation after random should be sanitized. (maybe sanitize random) +/* One iteration of hill climbing. Specifically, it first tries all + * valid parameter configurations w/ manhattan distance 1 and picks the best one + * failing that, it progressively tries candidates further and further away (up to #dim + 2) + * if it finds a candidate exceeding winnerInfo, it will repeat. Otherwise, it will stop the + * current stage of hill climbing. + * Each iteration of hill climbing proceeds in 2 'phases'. Phase 1 climbs according to + * the resultScore function, which is effectively a linear increase in reward until it reaches + * the constraint-satisfying value, it which point any excess results in only logarithmic reward. + * This aims to find some constraint-satisfying point. + * Phase 2 optimizes in accordance with what the original function sets out to maximize, with + * all feasible solutions valued over all infeasible solutions. + */ static winnerInfo_t climbOnce(const constraint_t target, const varInds_t* varArray, const int varLen, U8* memoTable, buffers_t buf, contexts_t ctx, const ZSTD_compressionParameters init) { - //distance maximizing selection? - //cparam - currently considered 'center' - //candidate - params to benchmark/results - //winner - best option found so far. + /* + * cparam - currently considered 'center' + * candidate - params to benchmark/results + * winner - best option found so far. + */ ZSTD_compressionParameters cparam = init; winnerInfo_t candidateInfo, winnerInfo; int better = 1; @@ -1506,14 +1506,12 @@ static winnerInfo_t climbOnce(const constraint_t target, cparam = winnerInfo.params; BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, buf.srcSize); candidateInfo.params = cparam; - //all dist-1 targets - //if we early end this, we should also randomize the order these are picked. + /* all dist-1 candidates */ for(i = 0; i < varLen; i++) { for(offset = -1; offset <= 1; offset += 2) { candidateInfo.params = cparam; - paramVaryOnce(varArray[i], offset, &candidateInfo.params); /* +1 */ + paramVaryOnce(varArray[i], offset, &candidateInfo.params); candidateInfo.params = sanitizeParams(candidateInfo.params); - //evaluate if(!ZSTD_isError(ZSTD_checkCParams(candidateInfo.params))) { int res = benchMemo(&candidateInfo.result, buf, ctx, @@ -1560,7 +1558,7 @@ static winnerInfo_t climbOnce(const constraint_t target, } } - if(!better) { //infeas -> feas -> stop. + if(!better) { /* infeas -> feas -> stop */ if(feas) { return winnerInfo; } feas = 1; @@ -1575,14 +1573,14 @@ static winnerInfo_t climbOnce(const constraint_t target, return winnerInfo; } -//optimizeForSize but with fixed strategy -//place to configure/filter out strategy specific parameters. +/* Optimizes for a fixed strategy */ -//flexible parameters: iterations of (failed?) climbing (or if we do non-random, maybe this is when everything is close to visitied) -//weight more on visit for bad results, less on good results/more on later results / ones with more failures. -//allocate memoTable here. -//only real use for paramTarget is to get the fixed values, right? -//maybe allow giving it a first init? +/* flexible parameters: iterations of (failed?) climbing (or if we do non-random, maybe this is when everything is close to visitied) + weight more on visit for bad results, less on good results/more on later results / ones with more failures. + allocate memoTable here. + only real use for paramTarget is to get the fixed values, right? + maybe allow giving it a first init? + */ static winnerInfo_t optimizeFixedStrategy( buffers_t buf, contexts_t ctx, const constraint_t target, ZSTD_compressionParameters paramTarget, @@ -1603,9 +1601,8 @@ static winnerInfo_t optimizeFixedStrategy( init = paramTarget; - while(i < tries) { //make i adjustable (user input?) depending on how much time they have. + while(i < tries) { DEBUGOUTPUT("Restart\n"); - //look into improving this to maximize distance from searched infeasible stuff / towards promising regions? randomConstrainedParams(&init, varNew, varLenNew, memoTable); candidateInfo = climbOnce(target, varNew, varLenNew, memoTable, buf, ctx, init); if(compareResultLT(winnerInfo.result, candidateInfo.result, target, buf.srcSize)) { @@ -1638,7 +1635,7 @@ static void freeBuffers(buffers_t b) { free(b.resPtrs); } -/* allocates buffer's arguments. returns success / failuere */ +/* allocates buffer's arguments. returns 0 = success / 1 = failuere */ static int createBuffers(buffers_t* buff, const char* const * const fileNamesTable, size_t nbFiles) { @@ -1748,16 +1745,25 @@ static void freeContexts(contexts_t ctx) { ZSTD_freeDCtx(ctx.dctx); } +/* Creates struct holding contexts and dictionary buffers. returns 0 on success, 1 on failure. */ static int createContexts(contexts_t* ctx, const char* dictFileName) { FILE* f; size_t readSize; ctx->cctx = ZSTD_createCCtx(); ctx->dctx = ZSTD_createDCtx(); + ctx->dictSize = 0; + ctx->dictBuffer = NULL; + + if(!ctx->cctx || !ctx->dctx) { + DISPLAY("context allocation error\n"); + freeContexts(*ctx); + return 1; + } + if(dictFileName == NULL) { - ctx->dictSize = 0; - ctx->dictBuffer = NULL; return 0; } + ctx->dictSize = UTIL_getFileSize(dictFileName); ctx->dictBuffer = malloc(ctx->dictSize); @@ -1786,8 +1792,8 @@ static int createContexts(contexts_t* ctx, const char* dictFileName) { return 0; } -//goes best, best-1, best+1, best-2, ... -//return 0 if nothing remaining +/* goes best, best-1, best+1, best-2, ... */ +/* return 0 if nothing remaining */ static int nextStrategy(const int currentStrategy, const int bestStrategy) { if(bestStrategy <= currentStrategy) { int candidate = 2 * bestStrategy - currentStrategy - 1; @@ -1828,7 +1834,10 @@ static ZSTD_compressionParameters maskParams(ZSTD_compressionParameters base, ZS } #define MAX_TRIES 8 -//optimize fixed strategy. +/* main fn called when using --optimize */ +/* Does strategy selection by benchmarking default compression levels + * then optimizes by strategy, starting with the best one and moving + * progressively moving further away by number */ static int optimizeForSize(const char* const * const fileNamesTable, const size_t nbFiles, const char* dictFileName, constraint_t target, ZSTD_compressionParameters paramTarget, int cLevel) { varInds_t varArray [NUM_PARAMS]; @@ -1905,7 +1914,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ goto _cleanUp; } - target.cSpeed = (U32)winner.result.cSpeed; //Maybe have a small bit of slack here, like x.99? + target.cSpeed = (U32)winner.result.cSpeed; g_targetConstraints = target; BMK_printWinner(stdout, cLevel, winner.result, winner.params, buf.srcSize); } @@ -1978,7 +1987,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ if(compareResultLT(winner.result, wc.result, target, buf.srcSize)) { winner = wc; } - //We could double back to increase search of 'better' strategies + st = nextStrategy(st, bestStrategy); tries--; } @@ -2088,7 +2097,7 @@ int main(int argc, const char** argv) int optimizerCLevel = 0; - constraint_t target = { 0, 0, (U32)-1 }; //0 for anything unset + constraint_t target = { 0, 0, (U32)-1 }; ZSTD_compressionParameters paramTarget = { 0, 0, 0, 0, 0, 0, 0 }; assert(argc>=1); /* for exename */ @@ -2250,7 +2259,7 @@ int main(int argc, const char** argv) /* load dictionary file (only applicable for optimizer rn) */ case 'D': - if(i == argc - 1) { //last argument, return error. + if(i == argc - 1) { /* last argument, return error. */ DISPLAY("Dictionary file expected but not given : %d\n", i); return 1; } else { From 3d230db85391aa5810abc6301035e9741c986765 Mon Sep 17 00:00:00 2001 From: George Lu Date: Mon, 6 Aug 2018 17:13:36 -0700 Subject: [PATCH 121/372] Change speed representation from floating point to integral --- programs/bench.c | 43 +++++++++++++++++++------------------------ programs/bench.h | 4 ++-- tests/paramgrill.c | 46 +++++++++++++++++----------------------------- 3 files changed, 38 insertions(+), 55 deletions(-) diff --git a/programs/bench.c b/programs/bench.c index b496caf27..49b317870 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -555,9 +555,7 @@ static BMK_return_t BMK_benchMemAdvancedNoAlloc( ratio = (double)(srcSize / intermediateResultCompress.result.result.sumOfReturn); { int const ratioAccuracy = (ratio < 10.) ? 3 : 2; - double const compressionSpeed = ((double)srcSize / intermediateResultCompress.result.result.nanoSecPerRun) * 1000; - int const cSpeedAccuracy = (compressionSpeed < 10.) ? 2 : 1; - results.result.cSpeed = compressionSpeed * 1000000; + results.result.cSpeed = (srcSize * TIMELOOP_NANOSEC / intermediateResultCompress.result.result.nanoSecPerRun); cSize = intermediateResultCompress.result.result.sumOfReturn; results.result.cSize = cSize; ratio = (double)srcSize / results.result.cSize; @@ -565,7 +563,7 @@ static BMK_return_t BMK_benchMemAdvancedNoAlloc( DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.*f),%6.*f MB/s\r", marks[markNb], displayName, (U32)srcSize, (U32)results.result.cSize, ratioAccuracy, ratio, - cSpeedAccuracy, compressionSpeed); + results.result.cSpeed < (10 MB) ? 2 : 1, (double)results.result.cSpeed / (1 MB)); } } @@ -579,16 +577,13 @@ static BMK_return_t BMK_benchMemAdvancedNoAlloc( { int const ratioAccuracy = (ratio < 10.) ? 3 : 2; - double const compressionSpeed = results.result.cSpeed / 1000000; - int const cSpeedAccuracy = (compressionSpeed < 10.) ? 2 : 1; - double const decompressionSpeed = ((double)srcSize / intermediateResultDecompress.result.result.nanoSecPerRun) * 1000; - results.result.dSpeed = decompressionSpeed * 1000000; + results.result.dSpeed = (srcSize * TIMELOOP_NANOSEC/ intermediateResultDecompress.result.result.nanoSecPerRun); markNb = (markNb+1) % NB_MARKS; DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.*f),%6.*f MB/s ,%6.1f MB/s \r", marks[markNb], displayName, (U32)srcSize, (U32)results.result.cSize, ratioAccuracy, ratio, - cSpeedAccuracy, compressionSpeed, - decompressionSpeed); + results.result.cSpeed < (10 MB) ? 2 : 1, (double)results.result.cSpeed / (1 MB), + (double)results.result.dSpeed / (1 MB)); } } } @@ -605,19 +600,20 @@ static BMK_return_t BMK_benchMemAdvancedNoAlloc( if(compressionResults.result.nanoSecPerRun == 0) { results.result.cSpeed = 0; } else { - results.result.cSpeed = (double)srcSize / compressionResults.result.nanoSecPerRun * TIMELOOP_NANOSEC; + results.result.cSpeed = srcSize * TIMELOOP_NANOSEC / compressionResults.result.nanoSecPerRun; } results.result.cSize = compressionResults.result.sumOfReturn; { int const ratioAccuracy = (ratio < 10.) ? 3 : 2; - double const compressionSpeed = results.result.cSpeed / 1000000; - int const cSpeedAccuracy = (compressionSpeed < 10.) ? 2 : 1; + results.result.cSpeed = (srcSize * TIMELOOP_NANOSEC / compressionResults.result.nanoSecPerRun); + cSize = compressionResults.result.sumOfReturn; + results.result.cSize = cSize; ratio = (double)srcSize / results.result.cSize; markNb = (markNb+1) % NB_MARKS; DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.*f),%6.*f MB/s\r", marks[markNb], displayName, (U32)srcSize, (U32)results.result.cSize, ratioAccuracy, ratio, - cSpeedAccuracy, compressionSpeed); + results.result.cSpeed < (10 MB) ? 2 : 1, (double)results.result.cSpeed / (1 MB)); } } if(adv->mode != BMK_compressOnly) { @@ -633,19 +629,18 @@ static BMK_return_t BMK_benchMemAdvancedNoAlloc( if(decompressionResults.result.nanoSecPerRun == 0) { results.result.dSpeed = 0; } else { - results.result.dSpeed = (double)srcSize / decompressionResults.result.nanoSecPerRun * TIMELOOP_NANOSEC; + results.result.dSpeed = srcSize * TIMELOOP_NANOSEC / decompressionResults.result.nanoSecPerRun; } - { int const ratioAccuracy = (ratio < 10.) ? 3 : 2; - double const compressionSpeed = results.result.cSpeed / 1000000; - int const cSpeedAccuracy = (compressionSpeed < 10.) ? 2 : 1; - double const decompressionSpeed = ((double)srcSize / decompressionResults.result.nanoSecPerRun) * 1000; - results.result.dSpeed = decompressionSpeed * 1000000; + + { + int const ratioAccuracy = (ratio < 10.) ? 3 : 2; + results.result.dSpeed = (srcSize * TIMELOOP_NANOSEC/ decompressionResults.result.nanoSecPerRun); markNb = (markNb+1) % NB_MARKS; DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.*f),%6.*f MB/s ,%6.1f MB/s \r", marks[markNb], displayName, (U32)srcSize, (U32)results.result.cSize, ratioAccuracy, ratio, - cSpeedAccuracy, compressionSpeed, - decompressionSpeed); + results.result.cSpeed < (10 MB) ? 2 : 1, (double)results.result.cSpeed / (1 MB), + (double)results.result.dSpeed / (1 MB)); } } } @@ -693,8 +688,8 @@ static BMK_return_t BMK_benchMemAdvancedNoAlloc( } /* CRC Checking */ if (displayLevel == 1) { /* hidden display mode -q, used by python speed benchmark */ - double const cSpeed = results.result.cSpeed / 1000000; - double const dSpeed = results.result.dSpeed / 1000000; + double const cSpeed = (double)results.result.cSpeed / (1 MB); + double const dSpeed = (double)results.result.dSpeed / (1 MB); if (adv->additionalParam) { DISPLAY("-%-3i%11i (%5.3f) %6.2f MB/s %6.1f MB/s %s (param=%d)\n", cLevel, (int)cSize, ratio, cSpeed, dSpeed, displayName, adv->additionalParam); } else { diff --git a/programs/bench.h b/programs/bench.h index 8baf33a0a..6247fa596 100644 --- a/programs/bench.h +++ b/programs/bench.h @@ -32,8 +32,8 @@ extern "C" { typedef struct { size_t cSize; - double cSpeed; /* bytes / sec */ - double dSpeed; + U64 cSpeed; /* bytes / sec */ + U64 dSpeed; size_t cMem; } BMK_result_t; diff --git a/tests/paramgrill.c b/tests/paramgrill.c index b8396c72e..5af3d88fa 100644 --- a/tests/paramgrill.c +++ b/tests/paramgrill.c @@ -249,18 +249,6 @@ static int feasible(const BMK_result_t results, const constraint_t target) { return (results.cSpeed >= target.cSpeed) && (results.dSpeed >= target.dSpeed) && (results.cMem <= target.cMem); } -#define EPSILON 0.001 -static int epsilonEqual(const double c1, const double c2) { - return MAX(c1/c2,c2/c1) < 1 + EPSILON; -} - -/* checks exact equivalence to 0, to stop compiler complaining fpeq */ -static int eqZero(const double c1) { - const double z1 = 0.0; - const double z2 = -0.0; - return !(memcmp(&c1, &z1, sizeof(double))) || !(memcmp(&c1, &z2, sizeof(double))); -} - /* hill climbing value for part 1 */ static double resultScore(const BMK_result_t res, const size_t srcSize, const constraint_t target) { double cs = 0., ds = 0., rt, cm = 0.; @@ -280,7 +268,7 @@ static double resultScore(const BMK_result_t res, const size_t srcSize, const co static int compareResultLT(const BMK_result_t result1, const BMK_result_t result2, const constraint_t target, size_t srcSize) { if(feasible(result1, target) && feasible(result2, target)) { return (result1.cSize > result2.cSize) || (result1.cSize == result2.cSize && result2.cSpeed > result1.cSpeed) - || (result1.cSize == result2.cSize && epsilonEqual(result2.cSpeed, result1.cSpeed) && result2.dSpeed > result1.dSpeed); + || (result1.cSize == result2.cSize && result2.cSpeed == result1.cSpeed && result2.dSpeed > result1.dSpeed); } return feasible(result2, target) || (!feasible(result1, target) && (resultScore(result1, srcSize, target) < resultScore(result2, srcSize, target))); @@ -661,7 +649,7 @@ static void BMK_printWinner(FILE* f, const U32 cLevel, const BMK_result_t result fprintf(f, "/* %s */ /* R:%5.3f at %5.1f MB/s - %5.1f MB/s */", - lvlstr, (double)srcSize / result.cSize, result.cSpeed / (1 << 20), result.dSpeed / (1 << 20)); + lvlstr, (double)srcSize / result.cSize, (double)result.cSpeed / (1 << 20), (double)result.dSpeed / (1 << 20)); if(TIMED) { fprintf(f, " - %1lu:%2lu:%05.2f", (unsigned long) minutes / 60,(unsigned long) minutes % 60, (double)(time - minutes * TIMELOOP_NANOSEC * 60ULL)/TIMELOOP_NANOSEC); } fprintf(f, "\n"); @@ -696,8 +684,8 @@ static void BMK_printWinners(FILE* f, const winnerInfo_t* winners, size_t srcSiz typedef struct { - double cSpeed_min; - double dSpeed_min; + U64 cSpeed_min; + U64 dSpeed_min; U32 windowLog_max; ZSTD_strategy strategy_max; } level_constraints_t; @@ -794,16 +782,16 @@ static int BMK_seed(winnerInfo_t* winners, const ZSTD_compressionParameters para /* too large compression speed difference for the compression benefit */ if (W_ratio > O_ratio) DISPLAY ("Compression Speed : %5.3f @ %4.1f MB/s vs %5.3f @ %4.1f MB/s : not enough for level %i\n", - W_ratio, testResult.cSpeed / 1000000, - O_ratio, winners[cLevel].result.cSpeed / 1000000., cLevel); + W_ratio, (double)testResult.cSpeed / 1000000, + O_ratio, (double)winners[cLevel].result.cSpeed / 1000000., cLevel); continue; } if (W_DSpeed_note < O_DSpeed_note ) { /* too large decompression speed difference for the compression benefit */ if (W_ratio > O_ratio) DISPLAY ("Decompression Speed : %5.3f @ %4.1f MB/s vs %5.3f @ %4.1f MB/s : not enough for level %i\n", - W_ratio, testResult.dSpeed / 1000000., - O_ratio, winners[cLevel].result.dSpeed / 1000000., cLevel); + W_ratio, (double)testResult.dSpeed / 1000000., + O_ratio, (double)winners[cLevel].result.dSpeed / 1000000., cLevel); continue; } @@ -1173,7 +1161,7 @@ static void BMK_benchOnce(ZSTD_CCtx* cctx, ZSTD_DCtx* dctx, const void* srcBuffe g_params = ZSTD_adjustCParams(g_params, srcSize, 0); BMK_benchParam1(&testResult, srcBuffer, srcSize, cctx, dctx, g_params); DISPLAY("Compression Ratio: %.3f Compress Speed: %.1f MB/s Decompress Speed: %.1f MB/s\n", (double)srcSize / testResult.cSize, - testResult.cSpeed / 1000000, testResult.dSpeed / 1000000); + (double)testResult.cSpeed / 1000000, (double)testResult.dSpeed / 1000000); return; } @@ -1355,20 +1343,20 @@ static int allBench(BMK_result_t* resultPtr, *resultPtr = benchres.result; /* calculate uncertainty in compression / decompression runs */ - if(eqZero(benchres.result.cSpeed)) { + if(benchres.result.cSpeed) { + loopDurationC = ((buf.srcSize * TIMELOOP_NANOSEC) / benchres.result.cSpeed); + uncertaintyConstantC = ((loopDurationC + (double)(2 * g_clockGranularity))/loopDurationC) * VARIANCE; + } else { loopDurationC = 0; uncertaintyConstantC = 3; - } else { - loopDurationC = (U64)((double)(buf.srcSize * TIMELOOP_NANOSEC) / benchres.result.cSpeed); - uncertaintyConstantC = ((loopDurationC + (double)(2 * g_clockGranularity))/loopDurationC) * VARIANCE; } - if(eqZero(benchres.result.dSpeed)) { + if(benchres.result.dSpeed) { + loopDurationD = ((buf.srcSize * TIMELOOP_NANOSEC) / benchres.result.dSpeed); + uncertaintyConstantD = ((loopDurationD + (double)(2 * g_clockGranularity))/loopDurationD) * VARIANCE; + } else { loopDurationD = 0; uncertaintyConstantD = 3; - } else { - loopDurationD = (U64)((double)(buf.srcSize * TIMELOOP_NANOSEC) / benchres.result.dSpeed); - uncertaintyConstantD = ((loopDurationD + (double)(2 * g_clockGranularity))/loopDurationD) * VARIANCE; } /* anything with worse ratio in feas is definitely worse, discard */ From 8278a49cb608af483b70f301742899f5b0b8ab7d Mon Sep 17 00:00:00 2001 From: George Lu Date: Mon, 6 Aug 2018 18:00:36 -0700 Subject: [PATCH 122/372] const srcPtrs --- programs/bench.c | 2 +- tests/paramgrill.c | 23 +++++++++++++++-------- 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/programs/bench.c b/programs/bench.c index 49b317870..874710c02 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -441,7 +441,7 @@ static BMK_return_t BMK_benchMemAdvancedNoAlloc( int displayLevel, const char* displayName, const BMK_advancedParams_t* adv) { size_t const blockSize = ((adv->blockSize>=32 && (adv->mode != BMK_decodeOnly)) ? adv->blockSize : srcSize) + (!srcSize); /* avoid div by 0 */ - BMK_return_t results = { { 0, 0., 0., 0 }, 0 } ; + BMK_return_t results = { { 0, 0, 0, 0 }, 0 } ; size_t const loadedCompressedSize = srcSize; size_t cSize = 0; double ratio = 0.; diff --git a/tests/paramgrill.c b/tests/paramgrill.c index 5af3d88fa..5d1a73081 100644 --- a/tests/paramgrill.c +++ b/tests/paramgrill.c @@ -250,6 +250,11 @@ static int feasible(const BMK_result_t results, const constraint_t target) { } /* hill climbing value for part 1 */ +/* Scoring here is a linear reward for all set constraints normalized between 0 to 1 + * (with 0 at 0 and 1 being fully fulfilling the constraint), summed with a logarithmic + * bonus to exceeding the constraint value. We also give linear ratio for compression ratio. + * The constant factors are experimental. + */ static double resultScore(const BMK_result_t res, const size_t srcSize, const constraint_t target) { double cs = 0., ds = 0., rt, cm = 0.; const double r1 = 1, r2 = 0.1, rtr = 0.5; @@ -291,7 +296,7 @@ const char* g_stratName[ZSTD_btultra+1] = { "ZSTD_greedy ", "ZSTD_lazy ", "ZSTD_lazy2 ", "ZSTD_btlazy2 ", "ZSTD_btopt ", "ZSTD_btultra "}; -/* benchParam but only takes in one file. */ +/* benchParam but only takes in one input buffer. */ static int BMK_benchParam1(BMK_result_t* resultPtr, const void* srcBuffer, size_t srcSize, @@ -324,8 +329,9 @@ static winnerInfo_t initWinnerInfo(ZSTD_compressionParameters p) { } typedef struct { + void* srcBuffer; size_t srcSize; - void** srcPtrs; + const void** srcPtrs; size_t* srcSizes; void** dstPtrs; size_t* dstCapacities; @@ -1605,7 +1611,7 @@ static winnerInfo_t optimizeFixedStrategy( static void freeBuffers(buffers_t b) { if(b.srcPtrs != NULL) { - free(b.srcPtrs[0]); + free(b.srcBuffer); } free(b.srcPtrs); free(b.srcSizes); @@ -1635,7 +1641,7 @@ static int createBuffers(buffers_t* buff, const char* const * const fileNamesTab U32 const maxNbBlocks = (U32) ((totalSizeToLoad + (blockSize-1)) / blockSize) + (U32)nbFiles; U32 blockNb = 0; - buff->srcPtrs = (void**)calloc(maxNbBlocks, sizeof(void*)); + buff->srcPtrs = (const void**)calloc(maxNbBlocks, sizeof(void*)); buff->srcSizes = (size_t*)malloc(maxNbBlocks * sizeof(size_t)); buff->dstPtrs = (void**)calloc(maxNbBlocks, sizeof(void*)); @@ -1651,7 +1657,8 @@ static int createBuffers(buffers_t* buff, const char* const * const fileNamesTab return 1; } - buff->srcPtrs[0] = malloc(benchedSize); + buff->srcBuffer = malloc(benchedSize); + buff->srcPtrs[0] = (const void*)buff->srcBuffer; buff->dstPtrs[0] = malloc(ZSTD_compressBound(benchedSize) + (maxNbBlocks * 1024)); buff->resPtrs[0] = malloc(benchedSize); @@ -1684,11 +1691,11 @@ static int createBuffers(buffers_t* buff, const char* const * const fileNamesTab if (fileSize + pos > benchedSize) fileSize = benchedSize - pos, nbFiles=n; /* buffer too small - stop after this file */ { - char* buffer = (char*)(buff->srcPtrs[0]); - size_t const readSize = fread((buffer)+pos, 1, (size_t)fileSize, f); + char* buffer = (char*)(buff->srcBuffer); + size_t const readSize = fread(((buffer)+pos), 1, (size_t)fileSize, f); size_t blocked = 0; while(blocked < readSize) { - buff->srcPtrs[blockNb] = (buffer) + (pos + blocked); + buff->srcPtrs[blockNb] = (const void*)((buffer) + (pos + blocked)); buff->srcSizes[blockNb] = blockSize; blocked += blockSize; blockNb++; From ad16a69408139d87cf44eb830b6749bf6066bee1 Mon Sep 17 00:00:00 2001 From: George Lu Date: Mon, 6 Aug 2018 18:37:55 -0700 Subject: [PATCH 123/372] Readability improvements, renaming --- tests/paramgrill.c | 131 +++++++++++++++++++++++++-------------------- 1 file changed, 72 insertions(+), 59 deletions(-) diff --git a/tests/paramgrill.c b/tests/paramgrill.c index 5d1a73081..c0679e88e 100644 --- a/tests/paramgrill.c +++ b/tests/paramgrill.c @@ -362,7 +362,7 @@ BMK_benchParam(BMK_result_t* resultPtr, *********************************************************/ static void BMK_initCCtx(ZSTD_CCtx* ctx, - const void* dictBuffer, size_t dictBufferSize, int cLevel, + const void* dictBuffer, const size_t dictBufferSize, const int cLevel, const ZSTD_compressionParameters* comprParams, const BMK_advancedParams_t* adv) { ZSTD_CCtx_reset(ctx); ZSTD_CCtx_resetParameters(ctx); @@ -389,7 +389,7 @@ static void BMK_initCCtx(ZSTD_CCtx* ctx, static void BMK_initDCtx(ZSTD_DCtx* dctx, - const void* dictBuffer, size_t dictBufferSize) { + const void* dictBuffer, const size_t dictBufferSize) { ZSTD_DCtx_reset(dctx); ZSTD_DCtx_loadDictionary(dctx, dictBuffer, dictBufferSize); } @@ -404,7 +404,7 @@ typedef struct { } BMK_initCCtxArgs; static size_t local_initCCtx(void* payload) { - BMK_initCCtxArgs* ag = (BMK_initCCtxArgs*)payload; + const BMK_initCCtxArgs* ag = (const BMK_initCCtxArgs*)payload; BMK_initCCtx(ag->ctx, ag->dictBuffer, ag->dictBufferSize, ag->cLevel, ag->comprParams, ag->adv); return 0; } @@ -416,7 +416,7 @@ typedef struct { } BMK_initDCtxArgs; static size_t local_initDCtx(void* payload) { - BMK_initDCtxArgs* ag = (BMK_initDCtxArgs*)payload; + const BMK_initDCtxArgs* ag = (const BMK_initDCtxArgs*)payload; BMK_initDCtx(ag->dctx, ag->dictBuffer, ag->dictBufferSize); return 0; } @@ -565,7 +565,7 @@ static BMK_return_t BMK_benchMemInvertible(buffers_t buf, contexts_t ctx, BMK_freeTimeState(timeStateDecompress); return results; } - results.result.cSpeed = ((double)srcSize / intermediateResultCompress.result.result.nanoSecPerRun) * TIMELOOP_NANOSEC; + results.result.cSpeed = (srcSize * TIMELOOP_NANOSEC) / intermediateResultCompress.result.result.nanoSecPerRun; results.result.cSize = intermediateResultCompress.result.result.sumOfReturn; } @@ -579,7 +579,7 @@ static BMK_return_t BMK_benchMemInvertible(buffers_t buf, contexts_t ctx, BMK_freeTimeState(timeStateDecompress); return results; } - results.result.dSpeed = ((double)srcSize / intermediateResultDecompress.result.result.nanoSecPerRun) * TIMELOOP_NANOSEC; + results.result.dSpeed = (srcSize * TIMELOOP_NANOSEC) / intermediateResultDecompress.result.result.nanoSecPerRun; } BMK_freeTimeState(timeStateCompress); @@ -597,7 +597,7 @@ static BMK_return_t BMK_benchMemInvertible(buffers_t buf, contexts_t ctx, if(compressionResults.result.nanoSecPerRun == 0) { results.result.cSpeed = 0; } else { - results.result.cSpeed = (double)srcSize / compressionResults.result.nanoSecPerRun * TIMELOOP_NANOSEC; + results.result.cSpeed = srcSize * TIMELOOP_NANOSEC / compressionResults.result.nanoSecPerRun; } results.result.cSize = compressionResults.result.sumOfReturn; } @@ -618,7 +618,7 @@ static BMK_return_t BMK_benchMemInvertible(buffers_t buf, contexts_t ctx, if(decompressionResults.result.nanoSecPerRun == 0) { results.result.dSpeed = 0; } else { - results.result.dSpeed = (double)srcSize / decompressionResults.result.nanoSecPerRun * TIMELOOP_NANOSEC; + results.result.dSpeed = srcSize * TIMELOOP_NANOSEC / decompressionResults.result.nanoSecPerRun; } } } @@ -628,39 +628,44 @@ static BMK_return_t BMK_benchMemInvertible(buffers_t buf, contexts_t ctx, return results; } -/* global winner used for display. */ -static winnerInfo_t g_winner = { { 0, 0, (size_t)-1, (size_t)-1 } , { 0, 0, 0, 0, 0, 0, ZSTD_fast } }; -static constraint_t g_targetConstraints; static void BMK_printWinner(FILE* f, const U32 cLevel, const BMK_result_t result, const ZSTD_compressionParameters params, const size_t srcSize) { - if(DEBUG || compareResultLT(g_winner.result, result, g_targetConstraints, srcSize)) { - char lvlstr[15] = "Custom Level"; - const U64 time = UTIL_clockSpanNano(g_time); - const U64 minutes = time / (60ULL * TIMELOOP_NANOSEC); + char lvlstr[15] = "Custom Level"; + const U64 time = UTIL_clockSpanNano(g_time); + const U64 minutes = time / (60ULL * TIMELOOP_NANOSEC); - if(DEBUG && compareResultLT(g_winner.result, result, g_targetConstraints, srcSize)) { + DISPLAY("\r%79s\r", ""); + + fprintf(f," {%3u,%3u,%3u,%3u,%3u,%3u, %s }, ", + params.windowLog, params.chainLog, params.hashLog, params.searchLog, params.searchLength, + params.targetLength, g_stratName[(U32)(params.strategy)]); + + if(cLevel != CUSTOM_LEVEL) { + snprintf(lvlstr, 15, " Level %2u ", cLevel); + } + + fprintf(f, + "/* %s */ /* R:%5.3f at %5.1f MB/s - %5.1f MB/s */", + lvlstr, (double)srcSize / result.cSize, (double)result.cSpeed / (1 << 20), (double)result.dSpeed / (1 << 20)); + + if(TIMED) { fprintf(f, " - %1lu:%2lu:%05.2f", (unsigned long) minutes / 60,(unsigned long) minutes % 60, (double)(time - minutes * TIMELOOP_NANOSEC * 60ULL)/TIMELOOP_NANOSEC); } + fprintf(f, "\n"); +} + +static void BMK_printWinnerOpt(FILE* f, const U32 cLevel, const BMK_result_t result, const ZSTD_compressionParameters params, const constraint_t targetConstraints, const size_t srcSize) +{ + /* global winner used for constraints */ + static winnerInfo_t g_winner = { { 0, 0, (size_t)-1, (size_t)-1 } , { 0, 0, 0, 0, 0, 0, ZSTD_fast } }; + + if(DEBUG || compareResultLT(g_winner.result, result, targetConstraints, srcSize)) { + if(DEBUG && compareResultLT(g_winner.result, result, targetConstraints, srcSize)) { DISPLAY("New Winner: \n"); } - DISPLAY("\r%79s\r", ""); + BMK_printWinner(f, cLevel, result, params, srcSize); - fprintf(f," {%3u,%3u,%3u,%3u,%3u,%3u, %s }, ", - params.windowLog, params.chainLog, params.hashLog, params.searchLog, params.searchLength, - params.targetLength, g_stratName[(U32)(params.strategy)]); - - if(cLevel != CUSTOM_LEVEL) { - snprintf(lvlstr, 15, " Level %2u ", cLevel); - } - - fprintf(f, - "/* %s */ /* R:%5.3f at %5.1f MB/s - %5.1f MB/s */", - lvlstr, (double)srcSize / result.cSize, (double)result.cSpeed / (1 << 20), (double)result.dSpeed / (1 << 20)); - - if(TIMED) { fprintf(f, " - %1lu:%2lu:%05.2f", (unsigned long) minutes / 60,(unsigned long) minutes % 60, (double)(time - minutes * TIMELOOP_NANOSEC * 60ULL)/TIMELOOP_NANOSEC); } - fprintf(f, "\n"); - - if(compareResultLT(g_winner.result, result, g_targetConstraints, srcSize)) { + if(compareResultLT(g_winner.result, result, targetConstraints, srcSize)) { BMK_translateAdvancedParams(params); g_winner.result = result; g_winner.params = params; @@ -950,12 +955,18 @@ static unsigned memoTableInd(const ZSTD_compressionParameters* ptr, const varInd unsigned ind = 0; for(i = 0; i < varyLen; i++) { switch(varyParams[i]) { - case wlog_ind: ind *= WLOG_RANGE; ind += ptr->windowLog - ZSTD_WINDOWLOG_MIN ; break; - case clog_ind: ind *= CLOG_RANGE; ind += ptr->chainLog - ZSTD_CHAINLOG_MIN ; break; - case hlog_ind: ind *= HLOG_RANGE; ind += ptr->hashLog - ZSTD_HASHLOG_MIN ; break; - case slog_ind: ind *= SLOG_RANGE; ind += ptr->searchLog - ZSTD_SEARCHLOG_MIN ; break; - case slen_ind: ind *= SLEN_RANGE; ind += ptr->searchLength - ZSTD_SEARCHLENGTH_MIN; break; - case tlen_ind: ind *= TLEN_RANGE; ind += tlen_inv(ptr->targetLength) - ZSTD_TARGETLENGTH_MIN; break; + case wlog_ind: ind *= WLOG_RANGE; ind += ptr->windowLog + - ZSTD_WINDOWLOG_MIN ; break; + case clog_ind: ind *= CLOG_RANGE; ind += ptr->chainLog + - ZSTD_CHAINLOG_MIN ; break; + case hlog_ind: ind *= HLOG_RANGE; ind += ptr->hashLog + - ZSTD_HASHLOG_MIN ; break; + case slog_ind: ind *= SLOG_RANGE; ind += ptr->searchLog + - ZSTD_SEARCHLOG_MIN ; break; + case slen_ind: ind *= SLEN_RANGE; ind += ptr->searchLength + - ZSTD_SEARCHLENGTH_MIN; break; + case tlen_ind: ind *= TLEN_RANGE; ind += tlen_inv(ptr->targetLength) + - ZSTD_TARGETLENGTH_MIN; break; } } return ind; @@ -976,8 +987,12 @@ static void memoTableIndInv(ZSTD_compressionParameters* ptr, const varInds_t* va } } -/* Initialize memotable, immediately mark redundant / obviously infeasible params as such */ -static void createMemoTable(U8* memoTable, ZSTD_compressionParameters paramConstraints, const constraint_t target, const varInds_t* varyParams, const int varyLen, const size_t srcSize) { +/* Initialize memoization table, which tracks and prevents repeated benchmarking + * of the same set of parameters. In addition, it is also used to immediately mark + * redundant / obviously non-optimal parameter configurations (e.g. wlog - 1 larger) + * than srcSize, clog > wlog, ... + */ +static void initMemoTable(U8* memoTable, ZSTD_compressionParameters paramConstraints, const constraint_t target, const varInds_t* varyParams, const int varyLen, const size_t srcSize) { size_t i; size_t arrayLen = memoTableLen(varyParams, varyLen); int cwFixed = !paramConstraints.chainLog || !paramConstraints.windowLog; @@ -985,6 +1000,7 @@ static void createMemoTable(U8* memoTable, ZSTD_compressionParameters paramConst int whFixed = !paramConstraints.windowLog || !paramConstraints.hashLog; int wFixed = !paramConstraints.windowLog; int j = 0; + assert(memoTable != NULL); memset(memoTable, 0, arrayLen); cParamZeroMin(¶mConstraints); @@ -994,7 +1010,7 @@ static void createMemoTable(U8* memoTable, ZSTD_compressionParameters paramConst memoTable[i] = 255; j++; } - if(wFixed && (1ULL << paramConstraints.windowLog) > srcSize) { + if(wFixed && (1ULL << (paramConstraints.windowLog - 1)) > srcSize) { memoTable[i] = 255; } /* nil out parameter sets equivalent to others. */ @@ -1033,7 +1049,7 @@ static void createMemoTable(U8* memoTable, ZSTD_compressionParameters paramConst } /* frees all allocated memotables */ -static void memoTableFreeAll(U8** mtAll) { +static void freeMemoTableArray(U8** mtAll) { int i; if(mtAll == NULL) { return; } for(i = 1; i <= (int)ZSTD_btultra; i++) { @@ -1056,10 +1072,10 @@ static U8** createMemoTableArray(ZSTD_compressionParameters paramConstraints, co const int varLenNew = sanitizeVarArray(varNew, varyLen, varyParams, i); mtAll[i] = malloc(sizeof(U8) * memoTableLen(varNew, varLenNew)); if(mtAll[i] == NULL) { - memoTableFreeAll(mtAll); + freeMemoTableArray(mtAll); return NULL; } - createMemoTable(mtAll[i], paramConstraints, target, varNew, varLenNew, srcSize); + initMemoTable(mtAll[i], paramConstraints, target, varNew, varLenNew, srcSize); } return mtAll; @@ -1451,7 +1467,7 @@ static int benchMemo(BMK_result_t* resultPtr, DISPLAY("Count: %d\n", bmcount); bmcount++; } - BMK_printWinner(stdout, CUSTOM_LEVEL, *resultPtr, cParams, buf.srcSize); + BMK_printWinnerOpt(stdout, CUSTOM_LEVEL, *resultPtr, cParams, target, buf.srcSize); if(res == BETTER_RESULT || feas) { memoTable[memind] = 255; @@ -1498,7 +1514,7 @@ static winnerInfo_t climbOnce(const constraint_t target, better = 0; DEBUGOUTPUT("Start\n"); cparam = winnerInfo.params; - BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, buf.srcSize); + BMK_printWinnerOpt(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, target, buf.srcSize); candidateInfo.params = cparam; /* all dist-1 candidates */ for(i = 0; i < varLen; i++) { @@ -1513,7 +1529,7 @@ static winnerInfo_t climbOnce(const constraint_t target, varArray, varLen, feas); if(res == BETTER_RESULT) { /* synonymous with better when called w/ infeasibleBM */ winnerInfo = candidateInfo; - BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, buf.srcSize); + BMK_printWinnerOpt(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, target, buf.srcSize); better = 1; if(compareResultLT(bestFeasible1.result, winnerInfo.result, target, buf.srcSize)) { bestFeasible1 = winnerInfo; @@ -1539,7 +1555,7 @@ static winnerInfo_t climbOnce(const constraint_t target, varArray, varLen, feas); if(res == BETTER_RESULT) { /* synonymous with better in this case*/ winnerInfo = candidateInfo; - BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, buf.srcSize); + BMK_printWinnerOpt(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, target, buf.srcSize); better = 1; if(compareResultLT(bestFeasible1.result, winnerInfo.result, target, buf.srcSize)) { bestFeasible1 = winnerInfo; @@ -1601,7 +1617,7 @@ static winnerInfo_t optimizeFixedStrategy( candidateInfo = climbOnce(target, varNew, varLenNew, memoTable, buf, ctx, init); if(compareResultLT(winnerInfo.result, candidateInfo.result, target, buf.srcSize)) { winnerInfo = candidateInfo; - BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, buf.srcSize); + BMK_printWinnerOpt(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, target, buf.srcSize); i = 0; } i++; @@ -1890,7 +1906,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ goto _cleanUp; } - createMemoTable(allMT[paramTarget.strategy], paramTarget, target, varNew, varLenNew, maxBlockSize); + initMemoTable(allMT[paramTarget.strategy], paramTarget, target, varNew, varLenNew, maxBlockSize); } else { allMT = createMemoTableArray(paramTarget, target, varArray, varLen, maxBlockSize); } @@ -1910,12 +1926,9 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ } target.cSpeed = (U32)winner.result.cSpeed; - g_targetConstraints = target; - BMK_printWinner(stdout, cLevel, winner.result, winner.params, buf.srcSize); + BMK_printWinnerOpt(stdout, cLevel, winner.result, winner.params, target, buf.srcSize); } - g_targetConstraints = target; - /* bench */ DISPLAY("\r%79s\r", ""); if(nbFiles == 1) { @@ -1946,7 +1959,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ ZSTD_compressionParameters CParams = ZSTD_getCParams(i, maxBlockSize, ctx.dictSize); CParams = maskParams(CParams, paramTarget); ec = BMK_benchParam(&candidate, buf, ctx, CParams); - BMK_printWinner(stdout, i, candidate, CParams, buf.srcSize); + BMK_printWinnerOpt(stdout, i, candidate, CParams, target, buf.srcSize); if(!ec && compareResultLT(winner.result, candidate, relaxTarget(target), buf.srcSize)) { winner.result = candidate; @@ -1956,7 +1969,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ } } - BMK_printWinner(stdout, CUSTOM_LEVEL, winner.result, winner.params, buf.srcSize); + BMK_printWinnerOpt(stdout, CUSTOM_LEVEL, winner.result, winner.params, target, buf.srcSize); BMK_translateAdvancedParams(winner.params); DEBUGOUTPUT("Real Opt\n"); /* start 'real' tests */ @@ -2000,7 +2013,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ goto _cleanUp; } /* end summary */ - BMK_printWinner(stdout, CUSTOM_LEVEL, winner.result, winner.params, buf.srcSize); + BMK_printWinnerOpt(stdout, CUSTOM_LEVEL, winner.result, winner.params, target, buf.srcSize); BMK_translateAdvancedParams(winner.params); DISPLAY("grillParams size - optimizer completed \n"); @@ -2008,7 +2021,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ _cleanUp: freeContexts(ctx); freeBuffers(buf); - memoTableFreeAll(allMT); + freeMemoTableArray(allMT); return ret; } From 6f480927af465d34ac9914789291094477c619bb Mon Sep 17 00:00:00 2001 From: George Lu Date: Tue, 7 Aug 2018 11:56:14 -0700 Subject: [PATCH 124/372] argument parsing cleanup + clarifying comment --- tests/paramgrill.c | 95 +++++++++++++++++++++++++++++++--------------- 1 file changed, 64 insertions(+), 31 deletions(-) diff --git a/tests/paramgrill.c b/tests/paramgrill.c index c0679e88e..a1d409d67 100644 --- a/tests/paramgrill.c +++ b/tests/paramgrill.c @@ -977,12 +977,18 @@ static void memoTableIndInv(ZSTD_compressionParameters* ptr, const varInds_t* va int i; for(i = varyLen - 1; i >= 0; i--) { switch(varyParams[i]) { - case wlog_ind: ptr->windowLog = ind % WLOG_RANGE + ZSTD_WINDOWLOG_MIN; ind /= WLOG_RANGE; break; - case clog_ind: ptr->chainLog = ind % CLOG_RANGE + ZSTD_CHAINLOG_MIN; ind /= CLOG_RANGE; break; - case hlog_ind: ptr->hashLog = ind % HLOG_RANGE + ZSTD_HASHLOG_MIN; ind /= HLOG_RANGE; break; - case slog_ind: ptr->searchLog = ind % SLOG_RANGE + ZSTD_SEARCHLOG_MIN; ind /= SLOG_RANGE; break; - case slen_ind: ptr->searchLength = ind % SLEN_RANGE + ZSTD_SEARCHLENGTH_MIN; ind /= SLEN_RANGE; break; - case tlen_ind: ptr->targetLength = tlen_table[(ind % TLEN_RANGE)]; ind /= TLEN_RANGE; break; + case wlog_ind: ptr->windowLog = ind % WLOG_RANGE + ZSTD_WINDOWLOG_MIN; + ind /= WLOG_RANGE; break; + case clog_ind: ptr->chainLog = ind % CLOG_RANGE + ZSTD_CHAINLOG_MIN; + ind /= CLOG_RANGE; break; + case hlog_ind: ptr->hashLog = ind % HLOG_RANGE + ZSTD_HASHLOG_MIN; + ind /= HLOG_RANGE; break; + case slog_ind: ptr->searchLog = ind % SLOG_RANGE + ZSTD_SEARCHLOG_MIN; + ind /= SLOG_RANGE; break; + case slen_ind: ptr->searchLength = ind % SLEN_RANGE + ZSTD_SEARCHLENGTH_MIN; + ind /= SLEN_RANGE; break; + case tlen_ind: ptr->targetLength = tlen_table[(ind % TLEN_RANGE)]; + ind /= TLEN_RANGE; break; } } } @@ -1135,13 +1141,20 @@ static ZSTD_compressionParameters randomParams(void) U32 validated = 0; while (!validated) { /* totally random entry */ - p.chainLog = (FUZ_rand(&g_rand) % (ZSTD_CHAINLOG_MAX+1 - ZSTD_CHAINLOG_MIN)) + ZSTD_CHAINLOG_MIN; - p.hashLog = (FUZ_rand(&g_rand) % (ZSTD_HASHLOG_MAX+1 - ZSTD_HASHLOG_MIN)) + ZSTD_HASHLOG_MIN; - p.searchLog = (FUZ_rand(&g_rand) % (ZSTD_SEARCHLOG_MAX+1 - ZSTD_SEARCHLOG_MIN)) + ZSTD_SEARCHLOG_MIN; - p.windowLog = (FUZ_rand(&g_rand) % (ZSTD_WINDOWLOG_MAX+1 - ZSTD_WINDOWLOG_MIN)) + ZSTD_WINDOWLOG_MIN; - p.searchLength=(FUZ_rand(&g_rand) % (ZSTD_SEARCHLENGTH_MAX+1 - ZSTD_SEARCHLENGTH_MIN)) + ZSTD_SEARCHLENGTH_MIN; + p.chainLog = (FUZ_rand(&g_rand) % (ZSTD_CHAINLOG_MAX+1 - ZSTD_CHAINLOG_MIN)) + + ZSTD_CHAINLOG_MIN; + p.hashLog = (FUZ_rand(&g_rand) % (ZSTD_HASHLOG_MAX+1 - ZSTD_HASHLOG_MIN)) + + ZSTD_HASHLOG_MIN; + p.searchLog = (FUZ_rand(&g_rand) % (ZSTD_SEARCHLOG_MAX+1 - ZSTD_SEARCHLOG_MIN)) + + ZSTD_SEARCHLOG_MIN; + p.windowLog = (FUZ_rand(&g_rand) % (ZSTD_WINDOWLOG_MAX+1 - ZSTD_WINDOWLOG_MIN)) + + ZSTD_WINDOWLOG_MIN; + p.searchLength=(FUZ_rand(&g_rand) % (ZSTD_SEARCHLENGTH_MAX+1 - ZSTD_SEARCHLENGTH_MIN)) + + ZSTD_SEARCHLENGTH_MIN; p.targetLength=(FUZ_rand(&g_rand) % (512)); + p.strategy = (ZSTD_strategy) (FUZ_rand(&g_rand) % (ZSTD_btultra +1)); + validated = !ZSTD_isError(ZSTD_checkCParams(p)); } return p; @@ -1657,6 +1670,8 @@ static int createBuffers(buffers_t* buff, const char* const * const fileNamesTab U32 const maxNbBlocks = (U32) ((totalSizeToLoad + (blockSize-1)) / blockSize) + (U32)nbFiles; U32 blockNb = 0; + memset(buff, 0, sizeof(buffers_t)); + buff->srcPtrs = (const void**)calloc(maxNbBlocks, sizeof(void*)); buff->srcSizes = (size_t*)malloc(maxNbBlocks * sizeof(size_t)); @@ -1760,6 +1775,7 @@ static void freeContexts(contexts_t ctx) { static int createContexts(contexts_t* ctx, const char* dictFileName) { FILE* f; size_t readSize; + U64 dictSize; ctx->cctx = ZSTD_createCCtx(); ctx->dctx = ZSTD_createDCtx(); ctx->dictSize = 0; @@ -1775,7 +1791,16 @@ static int createContexts(contexts_t* ctx, const char* dictFileName) { return 0; } - ctx->dictSize = UTIL_getFileSize(dictFileName); + dictSize = UTIL_getFileSize(dictFileName); + + if(dictSize == UTIL_FILESIZE_UNKNOWN) { + DISPLAY("Unable to get dictionary size\n"); + freeContexts(*ctx); + return 1; + } else { + ctx->dictSize = (size_t)dictSize; + } + ctx->dictBuffer = malloc(ctx->dictSize); f = fopen(dictFileName, "rb"); @@ -1848,7 +1873,15 @@ static ZSTD_compressionParameters maskParams(ZSTD_compressionParameters base, ZS /* main fn called when using --optimize */ /* Does strategy selection by benchmarking default compression levels * then optimizes by strategy, starting with the best one and moving - * progressively moving further away by number */ + * progressively moving further away by number + * args: + * fileNamesTable - list of files to benchmark + * nbFiles - length of fileNamesTable + * dictFileName - name of dictionary file if one, else NULL + * target - performance constraints (cSpeed, dSpeed, cMem) + * paramTarget - parameter constraints (i.e. restriction search space to where strategy = ZSTD_fast) + * cLevel - compression level to exceed (all solutions must be > lvl in cSpeed + ratio) + */ static int optimizeForSize(const char* const * const fileNamesTable, const size_t nbFiles, const char* dictFileName, constraint_t target, ZSTD_compressionParameters paramTarget, int cLevel) { varInds_t varArray [NUM_PARAMS]; @@ -2092,6 +2125,18 @@ static int badusage(const char* exename) return 1; } +#define PARSE_SUB_ARGS(stringLong, stringShort, variable) { if (longCommandWArg(&argument, stringLong) || longCommandWArg(&argument, stringShort)) { variable = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; } } +#define PARSE_CPARAMS(variable) \ +{ \ + PARSE_SUB_ARGS("windowLog=", "wlog=", variable.windowLog); \ + PARSE_SUB_ARGS("chainLog=" , "clog=", variable.chainLog); \ + PARSE_SUB_ARGS("hashLog=", "hlog=", variable.hashLog); \ + PARSE_SUB_ARGS("searchLog=" , "slog=", variable.searchLog); \ + PARSE_SUB_ARGS("searchLength=", "slen=", variable.searchLength); \ + PARSE_SUB_ARGS("targetLength=" , "tlen=", variable.targetLength); \ + PARSE_SUB_ARGS("strategy=", "strat=", variable.strategy); \ +} + int main(int argc, const char** argv) { int i, @@ -2127,17 +2172,11 @@ int main(int argc, const char** argv) if (longCommandWArg(&argument, "--optimize=")) { optimizer = 1; for ( ; ;) { - if (longCommandWArg(&argument, "windowLog=") || longCommandWArg(&argument, "wlog=")) { paramTarget.windowLog = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; } - if (longCommandWArg(&argument, "chainLog=") || longCommandWArg(&argument, "clog=")) { paramTarget.chainLog = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; } - if (longCommandWArg(&argument, "hashLog=") || longCommandWArg(&argument, "hlog=")) { paramTarget.hashLog = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; } - if (longCommandWArg(&argument, "searchLog=") || longCommandWArg(&argument, "slog=")) { paramTarget.searchLog = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; } - if (longCommandWArg(&argument, "searchLength=") || longCommandWArg(&argument, "slen=")) { paramTarget.searchLength = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; } - if (longCommandWArg(&argument, "targetLength=") || longCommandWArg(&argument, "tlen=")) { paramTarget.targetLength = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; } - if (longCommandWArg(&argument, "strategy=") || longCommandWArg(&argument, "strat=")) { paramTarget.strategy = (ZSTD_strategy)(readU32FromChar(&argument)); if (argument[0]==',') { argument++; continue; } else break; } - if (longCommandWArg(&argument, "compressionSpeed=") || longCommandWArg(&argument, "cSpeed=")) { target.cSpeed = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; } - if (longCommandWArg(&argument, "decompressionSpeed=") || longCommandWArg(&argument, "dSpeed=")) { target.dSpeed = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; } - if (longCommandWArg(&argument, "compressionMemory=") || longCommandWArg(&argument, "cMem=")) { target.cMem = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; } - if (longCommandWArg(&argument, "level=") || longCommandWArg(&argument, "lvl=")) { optimizerCLevel = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; } + PARSE_CPARAMS(paramTarget); + PARSE_SUB_ARGS("compressionSpeed=" , "cSpeed=", target.cSpeed); + PARSE_SUB_ARGS("decompressionSpeed=", "dSpeed=", target.dSpeed); + PARSE_SUB_ARGS("compressionMemory=" , "cMem=", target.cMem); + PARSE_SUB_ARGS("level=", "lvl=", optimizerCLevel); DISPLAY("invalid optimization parameter \n"); return 1; } @@ -2152,13 +2191,7 @@ int main(int argc, const char** argv) g_singleRun = 1; g_params = ZSTD_getCParams(2, g_blockSize, 0); for ( ; ;) { - if (longCommandWArg(&argument, "windowLog=") || longCommandWArg(&argument, "wlog=")) { g_params.windowLog = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; } - if (longCommandWArg(&argument, "chainLog=") || longCommandWArg(&argument, "clog=")) { g_params.chainLog = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; } - if (longCommandWArg(&argument, "hashLog=") || longCommandWArg(&argument, "hlog=")) { g_params.hashLog = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; } - if (longCommandWArg(&argument, "searchLog=") || longCommandWArg(&argument, "slog=")) { g_params.searchLog = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; } - if (longCommandWArg(&argument, "searchLength=") || longCommandWArg(&argument, "slen=")) { g_params.searchLength = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; } - if (longCommandWArg(&argument, "targetLength=") || longCommandWArg(&argument, "tlen=")) { g_params.targetLength = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; } - if (longCommandWArg(&argument, "strategy=") || longCommandWArg(&argument, "strat=")) { g_params.strategy = (ZSTD_strategy)(readU32FromChar(&argument)); if (argument[0]==',') { argument++; continue; } else break; } + PARSE_CPARAMS(g_params) if (longCommandWArg(&argument, "level=") || longCommandWArg(&argument, "lvl=")) { g_params = ZSTD_getCParams(readU32FromChar(&argument), g_blockSize, 0); if (argument[0]==',') { argument++; continue; } else break; } DISPLAY("invalid compression parameter \n"); return 1; From 0ece2e5cdc33d9e1499125820a85cae684c6deaa Mon Sep 17 00:00:00 2001 From: George Lu Date: Thu, 9 Aug 2018 11:38:09 -0700 Subject: [PATCH 125/372] Add consts + fix gcc-8 warnings --- tests/fullbench.c | 3 ++- tests/paramgrill.c | 48 ++++++++++++++++++++++++---------------------- 2 files changed, 27 insertions(+), 24 deletions(-) diff --git a/tests/fullbench.c b/tests/fullbench.c index 270cac86a..12c1e1ae4 100644 --- a/tests/fullbench.c +++ b/tests/fullbench.c @@ -514,9 +514,10 @@ static size_t benchMem(const void* src, size_t srcSize, U32 benchNb, int cLevel, /* benchmark loop */ { + void* dstBuffv = (void*)dstBuff; r = BMK_benchFunction(benchFunction, buff2, NULL, NULL, 1, &src, &srcSize, - (void **)&dstBuff, &dstBuffSize, NULL, g_nbIterations); + &dstBuffv, &dstBuffSize, NULL, g_nbIterations); if(r.error) { DISPLAY("ERROR %d ! ! \n", r.error); errorcode = r.error; diff --git a/tests/paramgrill.c b/tests/paramgrill.c index a1d409d67..eeafc8302 100644 --- a/tests/paramgrill.c +++ b/tests/paramgrill.c @@ -350,7 +350,7 @@ typedef struct { static int BMK_benchParam(BMK_result_t* resultPtr, - buffers_t buf, contexts_t ctx, + const buffers_t buf, const contexts_t ctx, const ZSTD_compressionParameters cParams) { BMK_return_t res = BMK_benchMem(buf.srcPtrs[0], buf.srcSize, buf.srcSizes, (unsigned)buf.nbBlocks, 0, &cParams, ctx.dictBuffer, ctx.dictSize, ctx.cctx, ctx.dctx, 0, "Files"); *resultPtr = res.result; @@ -482,13 +482,14 @@ static size_t local_defaultDecompress( * From Paramgrill End *********************************************************/ -/* Replicate function of benchMemAdvanced, but with pre-split src / dst buffers, with relevant info to invert it (compressedSizes) passed out. */ +/* Replicate functionality of benchMemAdvanced, but with pre-split src / dst buffers */ +/* The purpose is so that sufficient information is returned so that a decompression call to benchMemInvertible is possible */ /* BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, fileSizes, nbFiles, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); */ /* nbSeconds used in same way as in BMK_advancedParams_t, as nbIters when in iterMode */ /* if in decodeOnly, then srcPtr's will be compressed blocks, and uncompressedBlocks will be written to dstPtrs? */ /* dictionary nullable, nothing else though. */ -static BMK_return_t BMK_benchMemInvertible(buffers_t buf, contexts_t ctx, +static BMK_return_t BMK_benchMemInvertible(const buffers_t buf, const contexts_t ctx, const int cLevel, const ZSTD_compressionParameters* comprParams, const BMK_mode_t mode, const BMK_loopMode_t loopMode, const unsigned nbSeconds) { @@ -496,11 +497,11 @@ static BMK_return_t BMK_benchMemInvertible(buffers_t buf, contexts_t ctx, BMK_return_t results = { { 0, 0., 0., 0 }, 0 } ; const void *const *const srcPtrs = (const void *const *const)buf.srcPtrs; size_t const *const srcSizes = buf.srcSizes; - void** dstPtrs = buf.dstPtrs; - size_t* dstCapacities = buf.dstCapacities; - size_t* dstSizes = buf.dstSizes; - void** resPtrs = buf.resPtrs; - size_t* resSizes = buf.resSizes; + void** const dstPtrs = buf.dstPtrs; + size_t const *const dstCapacities = buf.dstCapacities; + size_t* const dstSizes = buf.dstSizes; + void** const resPtrs = buf.resPtrs; + size_t const *const resSizes = buf.resSizes; const void* dictBuffer = ctx.dictBuffer; const size_t dictBufferSize = ctx.dictSize; const size_t nbBlocks = buf.nbBlocks; @@ -1355,7 +1356,7 @@ int benchFiles(const char** fileNamesTable, int nbFiles) /* Benchmarking which stops when we are sufficiently sure the solution is infeasible / worse than the winner */ #define VARIANCE 1.1 static int allBench(BMK_result_t* resultPtr, - buffers_t buf, contexts_t ctx, + const buffers_t buf, const contexts_t ctx, const ZSTD_compressionParameters cParams, const constraint_t target, BMK_result_t* winnerResult, int feas) { @@ -1463,11 +1464,11 @@ static int allBench(BMK_result_t* resultPtr, /* Memoized benchmarking, won't benchmark anything which has already been benchmarked before. */ static int benchMemo(BMK_result_t* resultPtr, - buffers_t buf, contexts_t ctx, + const buffers_t buf, const contexts_t ctx, const ZSTD_compressionParameters cParams, const constraint_t target, - BMK_result_t* winnerResult, U8* memoTable, - const varInds_t* varyParams, const int varyLen, int feas) { + BMK_result_t* winnerResult, U8* const memoTable, + const varInds_t* varyParams, const int varyLen, const int feas) { static int bmcount = 0; size_t memind = memoTableInd(&cParams, varyParams, varyLen); int res; @@ -1502,8 +1503,8 @@ static int benchMemo(BMK_result_t* resultPtr, */ static winnerInfo_t climbOnce(const constraint_t target, const varInds_t* varArray, const int varLen, - U8* memoTable, - buffers_t buf, contexts_t ctx, + U8* const memoTable, + const buffers_t buf, const contexts_t ctx, const ZSTD_compressionParameters init) { /* * cparam - currently considered 'center' @@ -1605,11 +1606,11 @@ static winnerInfo_t climbOnce(const constraint_t target, maybe allow giving it a first init? */ static winnerInfo_t optimizeFixedStrategy( - buffers_t buf, contexts_t ctx, + const buffers_t buf, const contexts_t ctx, const constraint_t target, ZSTD_compressionParameters paramTarget, const ZSTD_strategy strat, const varInds_t* varArray, const int varLen, - U8* memoTable, const int tries) { + U8* const memoTable, const int tries) { int i = 0; varInds_t varNew[NUM_PARAMS]; int varLenNew = sanitizeVarArray(varNew, varLen, varArray, strat); @@ -1638,7 +1639,7 @@ static winnerInfo_t optimizeFixedStrategy( return winnerInfo; } -static void freeBuffers(buffers_t b) { +static void freeBuffers(const buffers_t b) { if(b.srcPtrs != NULL) { free(b.srcBuffer); } @@ -1659,8 +1660,8 @@ static void freeBuffers(buffers_t b) { } /* allocates buffer's arguments. returns 0 = success / 1 = failuere */ -static int createBuffers(buffers_t* buff, const char* const * const fileNamesTable, - size_t nbFiles) +static int createBuffers(buffers_t* const buff, const char* const * const fileNamesTable, + const size_t nbFiles) { size_t pos = 0; size_t n; @@ -1720,7 +1721,7 @@ static int createBuffers(buffers_t* buff, const char* const * const fileNamesTab DISPLAY("Loading %s... \r", fileNamesTable[n]); - if (fileSize + pos > benchedSize) fileSize = benchedSize - pos, nbFiles=n; /* buffer too small - stop after this file */ + if (fileSize + pos > benchedSize) fileSize = benchedSize - pos, n = nbFiles; /* buffer too small - stop after this file */ { char* buffer = (char*)(buff->srcBuffer); size_t const readSize = fread(((buffer)+pos), 1, (size_t)fileSize, f); @@ -1765,14 +1766,14 @@ static int createBuffers(buffers_t* buff, const char* const * const fileNamesTab return 0; } -static void freeContexts(contexts_t ctx) { +static void freeContexts(const contexts_t ctx) { free(ctx.dictBuffer); ZSTD_freeCCtx(ctx.cctx); ZSTD_freeDCtx(ctx.dctx); } /* Creates struct holding contexts and dictionary buffers. returns 0 on success, 1 on failure. */ -static int createContexts(contexts_t* ctx, const char* dictFileName) { +static int createContexts(contexts_t* const ctx, const char* dictFileName) { FILE* f; size_t readSize; U64 dictSize; @@ -2268,7 +2269,8 @@ int main(int argc, const char** argv) g_params.strategy = (ZSTD_strategy)readU32FromChar(&argument); continue; case 'L': - { int const cLevel = readU32FromChar(&argument); + { argument++; + int const cLevel = readU32FromChar(&argument); g_params = ZSTD_getCParams(cLevel, g_blockSize, 0); continue; } From bfe8392e23cbb561963451c154bde2a368fde926 Mon Sep 17 00:00:00 2001 From: George Lu Date: Thu, 9 Aug 2018 12:07:57 -0700 Subject: [PATCH 126/372] Remove ctx from benchMem --- programs/bench.c | 46 ++++++++++---------------------------- programs/bench.h | 4 ---- tests/paramgrill.c | 55 ++++++++++++++++++---------------------------- 3 files changed, 33 insertions(+), 72 deletions(-) diff --git a/programs/bench.c b/programs/bench.c index 874710c02..7662678ab 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -708,7 +708,6 @@ BMK_return_t BMK_benchMemAdvanced(const void* srcBuffer, size_t srcSize, const size_t* fileSizes, unsigned nbFiles, const int cLevel, const ZSTD_compressionParameters* comprParams, const void* dictBuffer, size_t dictBufferSize, - ZSTD_CCtx* ctx, ZSTD_DCtx* dctx, int displayLevel, const char* displayName, const BMK_advancedParams_t* adv) { @@ -730,6 +729,9 @@ BMK_return_t BMK_benchMemAdvanced(const void* srcBuffer, size_t srcSize, BMK_timedFnState_t* timeStateCompress = BMK_createTimeState(adv->nbSeconds); BMK_timedFnState_t* timeStateDecompress = BMK_createTimeState(adv->nbSeconds); + ZSTD_CCtx* ctx = ZSTD_createCCtx(); + ZSTD_DCtx* dctx = ZSTD_createDCtx(); + const size_t maxCompressedSize = dstCapacity ? dstCapacity : ZSTD_compressBound(srcSize) + (maxNbBlocks * 1024); void* const internalDstBuffer = dstBuffer ? NULL : malloc(maxCompressedSize); @@ -756,6 +758,9 @@ BMK_return_t BMK_benchMemAdvanced(const void* srcBuffer, size_t srcSize, BMK_freeTimeState(timeStateCompress); BMK_freeTimeState(timeStateDecompress); + ZSTD_freeCCtx(ctx); + ZSTD_freeDCtx(dctx); + free(internalDstBuffer); free(resultBuffer); @@ -781,7 +786,6 @@ BMK_return_t BMK_benchMem(const void* srcBuffer, size_t srcSize, const size_t* fileSizes, unsigned nbFiles, const int cLevel, const ZSTD_compressionParameters* comprParams, const void* dictBuffer, size_t dictBufferSize, - ZSTD_CCtx* ctx, ZSTD_DCtx* dctx, int displayLevel, const char* displayName) { const BMK_advancedParams_t adv = BMK_initAdvancedParams(); @@ -790,35 +794,9 @@ BMK_return_t BMK_benchMem(const void* srcBuffer, size_t srcSize, fileSizes, nbFiles, cLevel, comprParams, dictBuffer, dictBufferSize, - ctx, dctx, displayLevel, displayName, &adv); } -static BMK_return_t BMK_benchMemCtxless(const void* srcBuffer, size_t srcSize, - const size_t* fileSizes, unsigned nbFiles, - int cLevel, const ZSTD_compressionParameters* const comprParams, - const void* dictBuffer, size_t dictBufferSize, - int displayLevel, const char* displayName, - const BMK_advancedParams_t* const adv) -{ - BMK_return_t res; - ZSTD_CCtx* ctx = ZSTD_createCCtx(); - ZSTD_DCtx* dctx = ZSTD_createDCtx(); - if(ctx == NULL || dctx == NULL) { - EXM_THROW(12, BMK_return_t, "not enough memory for contexts"); - } - res = BMK_benchMemAdvanced(srcBuffer, srcSize, - NULL, 0, - fileSizes, nbFiles, - cLevel, comprParams, - dictBuffer, dictBufferSize, - ctx, dctx, - displayLevel, displayName, adv); - ZSTD_freeCCtx(ctx); - ZSTD_freeDCtx(dctx); - return res; -} - static size_t BMK_findMaxMem(U64 requiredMem) { size_t const step = 64 MB; @@ -859,12 +837,12 @@ static BMK_return_t BMK_benchCLevel(const void* srcBuffer, size_t benchedSize, if (displayLevel == 1 && !adv->additionalParam) DISPLAY("bench %s %s: input %u bytes, %u seconds, %u KB blocks\n", ZSTD_VERSION_STRING, ZSTD_GIT_COMMIT_STRING, (U32)benchedSize, adv->nbSeconds, (U32)(adv->blockSize>>10)); - res = BMK_benchMemCtxless(srcBuffer, benchedSize, - fileSizes, nbFiles, - cLevel, comprParams, - dictBuffer, dictBufferSize, - displayLevel, displayName, - adv); + res = BMK_benchMemAdvanced(srcBuffer, benchedSize, + NULL, 0, + fileSizes, nbFiles, + cLevel, comprParams, + dictBuffer, dictBufferSize, + displayLevel, displayName, adv); return res; } diff --git a/programs/bench.h b/programs/bench.h index 6247fa596..ad4ede1f0 100644 --- a/programs/bench.h +++ b/programs/bench.h @@ -126,8 +126,6 @@ BMK_return_t BMK_syntheticTest(int cLevel, double compressibility, * comprParams - basic compression parameters * dictBuffer - a dictionary if used, null otherwise * dictBufferSize - size of dictBuffer, 0 otherwise - * ctx - Compression Context (must be provided) - * dctx - Decompression Context (must be provided) * diplayLevel - see BMK_benchFiles * displayName - name used by display * return @@ -139,7 +137,6 @@ BMK_return_t BMK_benchMem(const void* srcBuffer, size_t srcSize, const size_t* fileSizes, unsigned nbFiles, const int cLevel, const ZSTD_compressionParameters* comprParams, const void* dictBuffer, size_t dictBufferSize, - ZSTD_CCtx* ctx, ZSTD_DCtx* dctx, int displayLevel, const char* displayName); /* See benchMem for normal parameter uses and return, see advancedParams_t for adv @@ -151,7 +148,6 @@ BMK_return_t BMK_benchMemAdvanced(const void* srcBuffer, size_t srcSize, const size_t* fileSizes, unsigned nbFiles, const int cLevel, const ZSTD_compressionParameters* comprParams, const void* dictBuffer, size_t dictBufferSize, - ZSTD_CCtx* ctx, ZSTD_DCtx* dctx, int displayLevel, const char* displayName, const BMK_advancedParams_t* adv); diff --git a/tests/paramgrill.c b/tests/paramgrill.c index eeafc8302..e530af735 100644 --- a/tests/paramgrill.c +++ b/tests/paramgrill.c @@ -300,10 +300,9 @@ const char* g_stratName[ZSTD_btultra+1] = { static int BMK_benchParam1(BMK_result_t* resultPtr, const void* srcBuffer, size_t srcSize, - ZSTD_CCtx* ctx, ZSTD_DCtx* dctx, const ZSTD_compressionParameters cParams) { - BMK_return_t res = BMK_benchMem(srcBuffer,srcSize, &srcSize, 1, 0, &cParams, NULL, 0, ctx, dctx, 0, "File"); + BMK_return_t res = BMK_benchMem(srcBuffer,srcSize, &srcSize, 1, 0, &cParams, NULL, 0, 0, "File"); *resultPtr = res.result; return res.error; } @@ -352,7 +351,7 @@ static int BMK_benchParam(BMK_result_t* resultPtr, const buffers_t buf, const contexts_t ctx, const ZSTD_compressionParameters cParams) { - BMK_return_t res = BMK_benchMem(buf.srcPtrs[0], buf.srcSize, buf.srcSizes, (unsigned)buf.nbBlocks, 0, &cParams, ctx.dictBuffer, ctx.dictSize, ctx.cctx, ctx.dctx, 0, "Files"); + BMK_return_t res = BMK_benchMem(buf.srcPtrs[0], buf.srcSize, buf.srcSizes, (unsigned)buf.nbBlocks, 0, &cParams, ctx.dictBuffer, ctx.dictSize, 0, "Files"); *resultPtr = res.result; return res.error; } @@ -724,14 +723,13 @@ static void BMK_init_level_constraints(int bytePerSec_level1) } static int BMK_seed(winnerInfo_t* winners, const ZSTD_compressionParameters params, - const void* srcBuffer, size_t srcSize, - ZSTD_CCtx* ctx, ZSTD_DCtx* dctx) + const void* srcBuffer, size_t srcSize) { BMK_result_t testResult; int better = 0; int cLevel; - BMK_benchParam1(&testResult, srcBuffer, srcSize, ctx, dctx, params); + BMK_benchParam1(&testResult, srcBuffer, srcSize, params); for (cLevel = 1; cLevel <= NB_LEVELS_TRACKED; cLevel++) { @@ -1104,8 +1102,7 @@ static BYTE* NB_TESTS_PLAYED(ZSTD_compressionParameters p) { static void playAround(FILE* f, winnerInfo_t* winners, ZSTD_compressionParameters params, - const void* srcBuffer, size_t srcSize, - ZSTD_CCtx* ctx, ZSTD_DCtx* dctx) + const void* srcBuffer, size_t srcSize) { int nbVariations = 0; UTIL_time_t const clockStart = UTIL_getTime(); @@ -1126,11 +1123,11 @@ static void playAround(FILE* f, winnerInfo_t* winners, /* test */ b = NB_TESTS_PLAYED(p); (*b)++; - if (!BMK_seed(winners, p, srcBuffer, srcSize, ctx, dctx)) continue; + if (!BMK_seed(winners, p, srcBuffer, srcSize)) continue; /* improvement found => search more */ BMK_printWinners(f, winners, srcSize); - playAround(f, winners, p, srcBuffer, srcSize, ctx, dctx); + playAround(f, winners, p, srcBuffer, srcSize); } } @@ -1177,31 +1174,30 @@ static void randomConstrainedParams(ZSTD_compressionParameters* pc, varInds_t* v static void BMK_selectRandomStart( FILE* f, winnerInfo_t* winners, - const void* srcBuffer, size_t srcSize, - ZSTD_CCtx* ctx, ZSTD_DCtx* dctx) + const void* srcBuffer, size_t srcSize) { U32 const id = FUZ_rand(&g_rand) % (NB_LEVELS_TRACKED+1); if ((id==0) || (winners[id].params.windowLog==0)) { /* use some random entry */ ZSTD_compressionParameters const p = ZSTD_adjustCParams(randomParams(), srcSize, 0); - playAround(f, winners, p, srcBuffer, srcSize, ctx, dctx); + playAround(f, winners, p, srcBuffer, srcSize); } else { - playAround(f, winners, winners[id].params, srcBuffer, srcSize, ctx, dctx); + playAround(f, winners, winners[id].params, srcBuffer, srcSize); } } -static void BMK_benchOnce(ZSTD_CCtx* cctx, ZSTD_DCtx* dctx, const void* srcBuffer, size_t srcSize) +static void BMK_benchOnce(const void* srcBuffer, size_t srcSize) { BMK_result_t testResult; g_params = ZSTD_adjustCParams(g_params, srcSize, 0); - BMK_benchParam1(&testResult, srcBuffer, srcSize, cctx, dctx, g_params); + BMK_benchParam1(&testResult, srcBuffer, srcSize, g_params); DISPLAY("Compression Ratio: %.3f Compress Speed: %.1f MB/s Decompress Speed: %.1f MB/s\n", (double)srcSize / testResult.cSize, (double)testResult.cSpeed / 1000000, (double)testResult.dSpeed / 1000000); return; } -static void BMK_benchFullTable(ZSTD_CCtx* cctx, ZSTD_DCtx* dctx, const void* srcBuffer, size_t srcSize) +static void BMK_benchFullTable(const void* srcBuffer, size_t srcSize) { ZSTD_compressionParameters params; winnerInfo_t winners[NB_LEVELS_TRACKED+1]; @@ -1220,7 +1216,7 @@ static void BMK_benchFullTable(ZSTD_CCtx* cctx, ZSTD_DCtx* dctx, const void* src /* baseline config for level 1 */ ZSTD_compressionParameters const l1params = ZSTD_getCParams(1, blockSize, 0); BMK_result_t testResult; - BMK_benchParam1(&testResult, srcBuffer, srcSize, cctx, dctx, l1params); + BMK_benchParam1(&testResult, srcBuffer, srcSize, l1params); BMK_init_level_constraints((int)((testResult.cSpeed * 31) / 32)); } @@ -1229,14 +1225,14 @@ static void BMK_benchFullTable(ZSTD_CCtx* cctx, ZSTD_DCtx* dctx, const void* src int i; for (i=0; i<=maxSeeds; i++) { params = ZSTD_getCParams(i, blockSize, 0); - BMK_seed(winners, params, srcBuffer, srcSize, cctx, dctx); + BMK_seed(winners, params, srcBuffer, srcSize); } } BMK_printWinners(f, winners, srcSize); /* start tests */ { const time_t grillStart = time(NULL); do { - BMK_selectRandomStart(f, winners, srcBuffer, srcSize, cctx, dctx); + BMK_selectRandomStart(f, winners, srcBuffer, srcSize); } while (BMK_timeSpan(grillStart) < g_grillDuration_s); } @@ -1248,21 +1244,12 @@ static void BMK_benchFullTable(ZSTD_CCtx* cctx, ZSTD_DCtx* dctx, const void* src fclose(f); } -static void BMK_benchMem_usingCCtx(ZSTD_CCtx* const cctx, ZSTD_DCtx* const dctx, const void* srcBuffer, size_t srcSize) +static void BMK_benchMemInit(const void* srcBuffer, size_t srcSize) { if (g_singleRun) - return BMK_benchOnce(cctx, dctx, srcBuffer, srcSize); + return BMK_benchOnce(srcBuffer, srcSize); else - return BMK_benchFullTable(cctx, dctx, srcBuffer, srcSize); -} - -static void BMK_benchMemCCtxInit(const void* srcBuffer, size_t srcSize) -{ - ZSTD_CCtx* const cctx = ZSTD_createCCtx(); - ZSTD_DCtx* const dctx = ZSTD_createDCtx(); - if (cctx==NULL || dctx==NULL) { DISPLAY("Context Creation failed \n"); exit(1); } - BMK_benchMem_usingCCtx(cctx, dctx, srcBuffer, srcSize); - ZSTD_freeCCtx(cctx); + return BMK_benchFullTable(srcBuffer, srcSize); } @@ -1280,7 +1267,7 @@ static int benchSample(void) /* bench */ DISPLAY("\r%79s\r", ""); DISPLAY("using %s %i%%: \n", name, (int)(g_compressibility*100)); - BMK_benchMemCCtxInit(origBuff, benchedSize); + BMK_benchMemInit(origBuff, benchedSize); free(origBuff); return 0; @@ -1338,7 +1325,7 @@ int benchFiles(const char** fileNamesTable, int nbFiles) /* bench */ DISPLAY("\r%79s\r", ""); DISPLAY("using %s : \n", inFileName); - BMK_benchMemCCtxInit(origBuff, benchedSize); + BMK_benchMemInit(origBuff, benchedSize); /* clean */ free(origBuff); From 51e71a5ec71091d6cb2e0e23b82d15eac9d1833f Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Thu, 9 Aug 2018 12:28:25 -0700 Subject: [PATCH 127/372] added zstdgrep documentation presenting `zstdgrep` limit regarding dictionary compression with workaround recommended by @tobwen (#1268) --- programs/README.md | 30 ++++++++++++++++++++++-------- 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/programs/README.md b/programs/README.md index 2833875e5..22a00409c 100644 --- a/programs/README.md +++ b/programs/README.md @@ -185,7 +185,7 @@ version is less than `128 MiB`). Compression Speed vs Ratio | Decompression Speed ---------------------------|--------------------- -![Compression Speed vs Ratio](../doc/images/ldmCspeed.png "Compression Speed vs Ratio") | ![Decompression Speed](../doc/images/ldmDspeed.png "Decompression Speed") +![Compression Speed vs Ratio](https://raw.githubusercontent.com/facebook/zstd/v1.3.3/doc/images/ldmCspeed.png "Compression Speed vs Ratio") | ![Decompression Speed](https://raw.githubusercontent.com/facebook/zstd/v1.3.3/doc/images/ldmDspeed.png "Decompression Speed") | Method | Compression ratio | Compression speed | Decompression speed | |:-------|------------------:|-------------------------:|---------------------------:| @@ -208,10 +208,24 @@ The below table illustrates this on the [Silesia compression corpus]. [Silesia compression corpus]: http://sun.aei.polsl.pl/~sdeor/index.php?page=silesia | Method | Compression ratio | Compression speed | Decompression speed | -|:-------|------------------:|-------------------------:|---------------------------:| -| `zstd -1` | `2.878` | `231.7 MB/s` | `594.4 MB/s` | -| `zstd -1 --long` | `2.929` | `106.5 MB/s` | `517.9 MB/s` | -| `zstd -5` | `3.274` | `77.1 MB/s` | `464.2 MB/s` | -| `zstd -5 --long` | `3.319` | `51.7 MB/s` | `371.9 MB/s` | -| `zstd -10` | `3.523` | `16.4 MB/s` | `489.2 MB/s` | -| `zstd -10 --long`| `3.566` | `16.2 MB/s` | `415.7 MB/s` | +|:-------|------------------:|------------------:|---------------------:| +| `zstd -1` | `2.878` | `231.7 MB/s` | `594.4 MB/s` | +| `zstd -1 --long` | `2.929` | `106.5 MB/s` | `517.9 MB/s` | +| `zstd -5` | `3.274` | `77.1 MB/s` | `464.2 MB/s` | +| `zstd -5 --long` | `3.319` | `51.7 MB/s` | `371.9 MB/s` | +| `zstd -10` | `3.523` | `16.4 MB/s` | `489.2 MB/s` | +| `zstd -10 --long`| `3.566` | `16.2 MB/s` | `415.7 MB/s` | + + +#### zstdgrep + +`zstdgrep` is a utility which makes it possible to `grep` directly a `.zst` compressed file. +It's used the same way as normal `grep`, for example : +`zstdgrep pattern file.zst` + +`zstdgrep` is _not_ compatible with dictionary compression. + +To search into a file compressed with a dictionary, +it's necessary to decompress it using `zstd` or `zstdcat`, +and then pipe the result to `grep`. For example : +`zstdcat -D dictionary -qc -- file.zst | grep pattern` From 79a35ac20d073271c46d14cf76d719f95a2a8f9c Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Thu, 9 Aug 2018 15:16:31 -0700 Subject: [PATCH 128/372] minor code comments improvements --- lib/compress/zstdmt_compress.c | 9 ++++++--- programs/fileio.c | 11 ++++------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/lib/compress/zstdmt_compress.c b/lib/compress/zstdmt_compress.c index d5193d52a..74f9dc29c 100644 --- a/lib/compress/zstdmt_compress.c +++ b/lib/compress/zstdmt_compress.c @@ -1080,9 +1080,9 @@ ZSTD_frameProgression ZSTDMT_getFrameProgression(ZSTDMT_CCtx* mtctx) { ZSTD_frameProgression fps; DEBUGLOG(6, "ZSTDMT_getFrameProgression"); + fps.ingested = mtctx->consumed + mtctx->inBuff.filled; fps.consumed = mtctx->consumed; fps.produced = mtctx->produced; - fps.ingested = mtctx->consumed + mtctx->inBuff.filled; { unsigned jobNb; unsigned lastJobNb = mtctx->nextJobID + mtctx->jobReady; assert(mtctx->jobReady <= 1); DEBUGLOG(6, "ZSTDMT_getFrameProgression: jobs: from %u to <%u (jobReady:%u)", @@ -1092,8 +1092,8 @@ ZSTD_frameProgression ZSTDMT_getFrameProgression(ZSTDMT_CCtx* mtctx) ZSTD_pthread_mutex_lock(&mtctx->jobs[wJobID].job_mutex); { size_t const cResult = mtctx->jobs[wJobID].cSize; size_t const produced = ZSTD_isError(cResult) ? 0 : cResult; - fps.consumed += mtctx->jobs[wJobID].consumed; fps.ingested += mtctx->jobs[wJobID].src.size; + fps.consumed += mtctx->jobs[wJobID].consumed; fps.produced += produced; } ZSTD_pthread_mutex_unlock(&mtctx->jobs[wJobID].job_mutex); @@ -1545,6 +1545,8 @@ static size_t ZSTDMT_createCompressionJob(ZSTDMT_CCtx* mtctx, size_t srcSize, ZS /*! ZSTDMT_flushProduced() : + * flush whatever data has been produced but not yet flushed in current job. + * move to next job if current one is fully flushed. * `output` : `pos` will be updated with amount of data flushed . * `blockToFlush` : if >0, the function will block and wait if there is no data available to flush . * @return : amount of data remaining within internal buffer, 0 if no more, 1 if unknown but > 0, or an error code */ @@ -1593,6 +1595,7 @@ static size_t ZSTDMT_flushProduced(ZSTDMT_CCtx* mtctx, ZSTD_outBuffer* output, u mtctx->jobs[wJobID].cSize += 4; /* can write this shared value, as worker is no longer active */ mtctx->jobs[wJobID].frameChecksumNeeded = 0; } + if (cSize > 0) { /* compression is ongoing or completed */ size_t const toFlush = MIN(cSize - mtctx->jobs[wJobID].dstFlushed, output->size - output->pos); DEBUGLOG(5, "ZSTDMT_flushProduced: Flushing %u bytes from job %u (completion:%u/%u, generated:%u)", @@ -1606,7 +1609,7 @@ static size_t ZSTDMT_flushProduced(ZSTDMT_CCtx* mtctx, ZSTD_outBuffer* output, u output->pos += toFlush; mtctx->jobs[wJobID].dstFlushed += toFlush; /* can write : this value is only used by mtctx */ - if ( (srcConsumed == srcSize) /* job completed */ + if ( (srcConsumed == srcSize) /* job is completed */ && (mtctx->jobs[wJobID].dstFlushed == cSize) ) { /* output buffer fully flushed => free this job position */ DEBUGLOG(5, "Job %u completed (%u bytes), moving to next one", mtctx->doneJobID, (U32)mtctx->jobs[wJobID].dstFlushed); diff --git a/programs/fileio.c b/programs/fileio.c index 85367fdfc..c1587f8f8 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -727,11 +727,6 @@ static unsigned long long FIO_compressLz4Frame(cRess_t* ress, #endif -/*! FIO_compressFilename_internal() : - * same as FIO_compressFilename_extRess(), with `ress.desFile` already opened. - * @return : 0 : compression completed correctly, - * 1 : missing or pb opening srcFileName - */ static unsigned long long FIO_compressZstdFrame(const cRess_t* ressPtr, const char* srcFileName, U64 fileSize, @@ -763,7 +758,8 @@ FIO_compressZstdFrame(const cRess_t* ressPtr, directive = ZSTD_e_end; result = 1; - while (inBuff.pos != inBuff.size || (directive == ZSTD_e_end && result != 0)) { + while ((inBuff.pos != inBuff.size) /* input buffer must be entirely ingested */ + || (directive == ZSTD_e_end && result != 0) ) { ZSTD_outBuffer outBuff = { ress.dstBuffer, ress.dstBufferSize, 0 }; CHECK_V(result, ZSTD_compress_generic(ress.cctx, &outBuff, &inBuff, directive)); @@ -786,7 +782,8 @@ FIO_compressZstdFrame(const cRess_t* ressPtr, (U32)(zfp.consumed >> 20), (U32)(zfp.produced >> 20), cShare ); - } else { /* g_displayLevel == 2 */ + } else { + assert(g_displayLevel == 2); DISPLAYLEVEL(2, "\rRead : %u ", (U32)(zfp.consumed >> 20)); if (fileSize != UTIL_FILESIZE_UNKNOWN) DISPLAYLEVEL(2, "/ %u ", (U32)(fileSize >> 20)); From 2dd76037be46a8b7589ed95cece4cff5df62330c Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Thu, 9 Aug 2018 15:51:30 -0700 Subject: [PATCH 129/372] zstd cli can increase level when input is too slow --- lib/compress/zstd_compress.c | 1 + lib/compress/zstdmt_compress.c | 1 + lib/zstd.h | 1 + programs/fileio.c | 29 ++++++++++++++++++++++++----- 4 files changed, 27 insertions(+), 5 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index ed3aab871..2121fe749 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -900,6 +900,7 @@ ZSTD_frameProgression ZSTD_getFrameProgression(const ZSTD_CCtx* cctx) fp.ingested = cctx->consumedSrcSize + buffered; fp.consumed = cctx->consumedSrcSize; fp.produced = cctx->producedCSize; + fp.currentJobID = 0; return fp; } } diff --git a/lib/compress/zstdmt_compress.c b/lib/compress/zstdmt_compress.c index 74f9dc29c..d2f06e4ee 100644 --- a/lib/compress/zstdmt_compress.c +++ b/lib/compress/zstdmt_compress.c @@ -1083,6 +1083,7 @@ ZSTD_frameProgression ZSTDMT_getFrameProgression(ZSTDMT_CCtx* mtctx) fps.ingested = mtctx->consumed + mtctx->inBuff.filled; fps.consumed = mtctx->consumed; fps.produced = mtctx->produced; + fps.currentJobID = mtctx->nextJobID; { unsigned jobNb; unsigned lastJobNb = mtctx->nextJobID + mtctx->jobReady; assert(mtctx->jobReady <= 1); DEBUGLOG(6, "ZSTDMT_getFrameProgression: jobs: from %u to <%u (jobReady:%u)", diff --git a/lib/zstd.h b/lib/zstd.h index 0c20bb768..edd0079c9 100644 --- a/lib/zstd.h +++ b/lib/zstd.h @@ -735,6 +735,7 @@ typedef struct { unsigned long long ingested; unsigned long long consumed; unsigned long long produced; + unsigned currentJobID; } ZSTD_frameProgression; /* ZSTD_getFrameProgression(): diff --git a/programs/fileio.c b/programs/fileio.c index c1587f8f8..b62e25703 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -737,6 +737,9 @@ FIO_compressZstdFrame(const cRess_t* ressPtr, FILE* const dstFile = ress.dstFile; U64 compressedfilesize = 0; ZSTD_EndDirective directive = ZSTD_e_continue; + unsigned inputBlocked = 0; + unsigned lastJobID = 0; + DISPLAYLEVEL(6, "compression using zstd format \n"); /* init */ @@ -747,7 +750,7 @@ FIO_compressZstdFrame(const cRess_t* ressPtr, /* Main compression loop */ do { - size_t result; + size_t stillToFlush; /* Fill input Buffer */ size_t const inSize = fread(ress.srcBuffer, (size_t)1, ress.srcBufferSize, srcFile); ZSTD_inBuffer inBuff = { ress.srcBuffer, inSize, 0 }; @@ -757,14 +760,18 @@ FIO_compressZstdFrame(const cRess_t* ressPtr, if ((inSize == 0) || (*readsize == fileSize)) directive = ZSTD_e_end; - result = 1; + stillToFlush = 1; while ((inBuff.pos != inBuff.size) /* input buffer must be entirely ingested */ - || (directive == ZSTD_e_end && result != 0) ) { + || (directive == ZSTD_e_end && stillToFlush != 0) ) { + size_t const oldIPos = inBuff.pos; ZSTD_outBuffer outBuff = { ress.dstBuffer, ress.dstBufferSize, 0 }; - CHECK_V(result, ZSTD_compress_generic(ress.cctx, &outBuff, &inBuff, directive)); + CHECK_V(stillToFlush, ZSTD_compress_generic(ress.cctx, &outBuff, &inBuff, directive)); + + /* count stats */ + if (oldIPos == inBuff.pos) inputBlocked++; /* Write compressed stream */ - DISPLAYLEVEL(6, "ZSTD_compress_generic(end:%u) => intput pos(%u)<=(%u)size ; output generated %u bytes \n", + DISPLAYLEVEL(6, "ZSTD_compress_generic(end:%u) => input pos(%u)<=(%u)size ; output generated %u bytes \n", (U32)directive, (U32)inBuff.pos, (U32)inBuff.size, (U32)outBuff.pos); if (outBuff.pos) { size_t const sizeCheck = fwrite(ress.dstBuffer, 1, outBuff.pos, dstFile); @@ -775,6 +782,18 @@ FIO_compressZstdFrame(const cRess_t* ressPtr, if (READY_FOR_UPDATE()) { ZSTD_frameProgression const zfp = ZSTD_getFrameProgression(ress.cctx); double const cShare = (double)zfp.produced / (zfp.consumed + !zfp.consumed/*avoid div0*/) * 100; + + /* check input speed */ + if (zfp.currentJobID >= lastJobID+2) { + if (inputBlocked<=1) { /* small tolerance */ + DISPLAYLEVEL(6, "input is never blocked => input is too slow \n"); + compressionLevel++; + ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_compressionLevel, (unsigned)compressionLevel); + } + lastJobID = zfp.currentJobID; + inputBlocked = 0; + } + if (g_displayLevel >= 3) { DISPLAYUPDATE(3, "\r(L%i) Buffered :%4u MB - Consumed :%4u MB - Compressed :%4u MB => %.2f%%", compressionLevel, From 754942cb799fc81a3c590733a1b77958bf74d818 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Thu, 9 Aug 2018 15:57:19 -0700 Subject: [PATCH 130/372] fixed assert() condition --- programs/fileio.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/programs/fileio.c b/programs/fileio.c index c1587f8f8..39b2c741c 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -783,7 +783,7 @@ FIO_compressZstdFrame(const cRess_t* ressPtr, (U32)(zfp.produced >> 20), cShare ); } else { - assert(g_displayLevel == 2); + /* g_displayLevel <= 2; only display notifications if == 2; */ DISPLAYLEVEL(2, "\rRead : %u ", (U32)(zfp.consumed >> 20)); if (fileSize != UTIL_FILESIZE_UNKNOWN) DISPLAYLEVEL(2, "/ %u ", (U32)(fileSize >> 20)); From 3ac2c22485ab5508f47e3eab642b787af0e68b5f Mon Sep 17 00:00:00 2001 From: George Lu Date: Thu, 9 Aug 2018 16:38:32 -0700 Subject: [PATCH 131/372] Reorder declaration --- tests/paramgrill.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/paramgrill.c b/tests/paramgrill.c index e530af735..bcb6bcae3 100644 --- a/tests/paramgrill.c +++ b/tests/paramgrill.c @@ -2256,8 +2256,9 @@ int main(int argc, const char** argv) g_params.strategy = (ZSTD_strategy)readU32FromChar(&argument); continue; case 'L': - { argument++; - int const cLevel = readU32FromChar(&argument); + { int cLevel; + argument++; + cLevel = readU32FromChar(&argument); g_params = ZSTD_getCParams(cLevel, g_blockSize, 0); continue; } From 9d26cb6a75dc3d19073c2973d7e3dc0ba0ba2fcc Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Thu, 9 Aug 2018 17:44:30 -0700 Subject: [PATCH 132/372] slow down faster when output speed is limited --- programs/fileio.c | 56 +++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 47 insertions(+), 9 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index fe3c13527..c12126168 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -737,8 +737,13 @@ FIO_compressZstdFrame(const cRess_t* ressPtr, FILE* const dstFile = ress.dstFile; U64 compressedfilesize = 0; ZSTD_EndDirective directive = ZSTD_e_continue; + + typedef enum { noChange, slower, faster } speedChange_e; + speedChange_e speedChange = noChange; unsigned inputBlocked = 0; unsigned lastJobID = 0; + unsigned long long lastProduced = 0; + unsigned long long lastFlushedSize = 0; DISPLAYLEVEL(6, "compression using zstd format \n"); @@ -763,6 +768,7 @@ FIO_compressZstdFrame(const cRess_t* ressPtr, stillToFlush = 1; while ((inBuff.pos != inBuff.size) /* input buffer must be entirely ingested */ || (directive == ZSTD_e_end && stillToFlush != 0) ) { + size_t const oldIPos = inBuff.pos; ZSTD_outBuffer outBuff = { ress.dstBuffer, ress.dstBufferSize, 0 }; CHECK_V(stillToFlush, ZSTD_compress_generic(ress.cctx, &outBuff, &inBuff, directive)); @@ -779,23 +785,55 @@ FIO_compressZstdFrame(const cRess_t* ressPtr, EXM_THROW(25, "Write error : cannot write compressed block"); compressedfilesize += outBuff.pos; } + + /* display notification; and adapt compression level */ if (READY_FOR_UPDATE()) { ZSTD_frameProgression const zfp = ZSTD_getFrameProgression(ress.cctx); double const cShare = (double)zfp.produced / (zfp.consumed + !zfp.consumed/*avoid div0*/) * 100; - /* check input speed */ - if (zfp.currentJobID >= lastJobID+2) { - if (inputBlocked<=1) { /* small tolerance */ - DISPLAYLEVEL(6, "input is never blocked => input is too slow \n"); - compressionLevel++; - ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_compressionLevel, (unsigned)compressionLevel); + /* check output speed */ + if (zfp.currentJobID > 0) { + unsigned long long newlyProduced = zfp.produced - lastProduced; + unsigned long long newlyFlushed = compressedfilesize - lastFlushedSize; + assert(zfp.produced >= lastProduced); + if (newlyProduced == 0) { + DISPLAYLEVEL(6, "no more data compression generation => buffers are full, compression waiting => output (or input) too slow \n") + speedChange = slower; } - lastJobID = zfp.currentJobID; - inputBlocked = 0; + + if ( (newlyProduced > (newlyFlushed * 9 / 8)) + && (stillToFlush > ZSTD_BLOCKSIZE_MAX) ) { + DISPLAYLEVEL(6, "production faster than flushing (%llu > %llu) \n", newlyProduced, newlyFlushed); + speedChange = slower; + } + lastProduced = zfp.produced; + lastFlushedSize = compressedfilesize; } + /* course correct only if there is at least one job completed */ + if (zfp.currentJobID > lastJobID) { + DISPLAYLEVEL(6, "compression level adaptation check \n") + + /* check input speed */ + if (zfp.currentJobID > g_nbWorkers+1) { /* warm up period, to fill all workers */ + if (inputBlocked <= 1) { /* small tolerance */ + DISPLAYLEVEL(6, "input is never blocked => input is too slow \n"); + speedChange = slower; + } + inputBlocked = 0; + } + + if (speedChange == slower) { + DISPLAYLEVEL(6, "slower speed , higher compression \n") + compressionLevel ++; + ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_compressionLevel, (unsigned)compressionLevel); + speedChange = noChange; + } + lastJobID = zfp.currentJobID; + } /* if (zfp.currentJobID > lastJobID) */ + if (g_displayLevel >= 3) { - DISPLAYUPDATE(3, "\r(L%i) Buffered :%4u MB - Consumed :%4u MB - Compressed :%4u MB => %.2f%%", + DISPLAYUPDATE(3, "\r(L%i) Buffered :%4u MB - Consumed :%4u MB - Compressed :%4u MB => %.2f%% ", compressionLevel, (U32)((zfp.ingested - zfp.consumed) >> 20), (U32)(zfp.consumed >> 20), From 681a382eea63e8f415d4aa40c65d4db973f5cd0c Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Fri, 10 Aug 2018 12:25:52 -0700 Subject: [PATCH 133/372] added rateLimiter.py, by @felixhandte this rate limiter avoid the problem of `pv` which "catch up" after a blocked period instead of preserving a constant speed cap. --- tests/rateLimiter.py | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100755 tests/rateLimiter.py diff --git a/tests/rateLimiter.py b/tests/rateLimiter.py new file mode 100755 index 000000000..559f2275a --- /dev/null +++ b/tests/rateLimiter.py @@ -0,0 +1,33 @@ +#! /usr/bin/env python3 + +# ################################################################ +# Copyright (c) 2018-present, Facebook, Inc. +# All rights reserved. +# +# This source code is licensed under both the BSD-style license (found in the +# LICENSE file in the root directory of this source tree) and the GPLv2 (found +# in the COPYING file in the root directory of this source tree). +# ########################################################################## + +# Rate limiter, replacement for pv +# this rate limiter does not "catch up" after a blocking period +# Limitations: +# - only accepts limit speed in MB/s + +import sys +import time + +rate = float(sys.argv[1]) * 1024 * 1024 +start = time.time() +total_read = 0 + +buf = " " +while len(buf): + now = time.time() + to_read = max(int(rate * (now - start) - total_read), 1) + buf = sys.stdin.read(to_read) + write_start = time.time() + sys.stdout.write(buf) + write_end = time.time() + start += write_end - write_start + total_read += len(buf) From a996b1fd2d74c50a77e8ed0303f3ca0a87e45b2f Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Fri, 10 Aug 2018 17:39:00 -0700 Subject: [PATCH 134/372] fixed rate limited for high speed --- tests/rateLimiter.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/tests/rateLimiter.py b/tests/rateLimiter.py index 559f2275a..134ef8971 100755 --- a/tests/rateLimiter.py +++ b/tests/rateLimiter.py @@ -1,4 +1,4 @@ -#! /usr/bin/env python3 +#!/usr/bin/env python3 # ################################################################ # Copyright (c) 2018-present, Facebook, Inc. @@ -17,7 +17,9 @@ import sys import time -rate = float(sys.argv[1]) * 1024 * 1024 +MB = 1024 * 1024 +rate = float(sys.argv[1]) * MB +rate *= 1.25 # compensation for excluding write time (experimentally determined) start = time.time() total_read = 0 @@ -25,9 +27,11 @@ buf = " " while len(buf): now = time.time() to_read = max(int(rate * (now - start) - total_read), 1) - buf = sys.stdin.read(to_read) + max_buf_size = 1 * MB + to_read = min(to_read, max_buf_size) + buf = sys.stdin.buffer.read(to_read) write_start = time.time() - sys.stdout.write(buf) + sys.stdout.buffer.write(buf) write_end = time.time() - start += write_end - write_start + start += write_end - write_start # exclude write delay total_read += len(buf) From e7a49c668306e9fc5b9f836c9eac130dc0516163 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Sat, 11 Aug 2018 20:48:06 -0700 Subject: [PATCH 135/372] introduced command --adapt --- lib/compress/zstd_compress.c | 1 + lib/compress/zstdmt_compress.c | 25 ++++++++----- programs/fileio.c | 66 ++++++++++++++++++++++++++-------- programs/fileio.h | 1 + programs/zstd.1.md | 29 ++++++++------- programs/zstdcli.c | 14 ++++---- 6 files changed, 94 insertions(+), 42 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 2121fe749..1412c1d6a 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -3704,6 +3704,7 @@ size_t ZSTD_compress_generic (ZSTD_CCtx* cctx, || (endOp == ZSTD_e_end && flushMin == 0) ) { /* compression completed */ ZSTD_CCtx_reset(cctx); } + DEBUGLOG(5, "completed ZSTD_compress_generic delegating to ZSTDMT_compressStream_generic"); return flushMin; } } #endif diff --git a/lib/compress/zstdmt_compress.c b/lib/compress/zstdmt_compress.c index d2f06e4ee..49502bd0d 100644 --- a/lib/compress/zstdmt_compress.c +++ b/lib/compress/zstdmt_compress.c @@ -249,8 +249,8 @@ static buffer_t ZSTDMT_resizeBuffer(ZSTDMT_bufferPool* bufPool, buffer_t buffer) /* store buffer for later re-use, up to pool capacity */ static void ZSTDMT_releaseBuffer(ZSTDMT_bufferPool* bufPool, buffer_t buf) { - if (buf.start == NULL) return; /* compatible with release on NULL */ DEBUGLOG(5, "ZSTDMT_releaseBuffer"); + if (buf.start == NULL) return; /* compatible with release on NULL */ ZSTD_pthread_mutex_lock(&bufPool->poolMutex); if (bufPool->nbBuffers < bufPool->totalBuffers) { bufPool->bTable[bufPool->nbBuffers++] = buf; /* stored for later use */ @@ -541,6 +541,7 @@ static void ZSTDMT_serialState_update(serialState_t* serialState, /* Wait for our turn */ ZSTD_PTHREAD_MUTEX_LOCK(&serialState->mutex); while (serialState->nextJobID < jobID) { + DEBUGLOG(5, "wait for serialState->cond"); ZSTD_pthread_cond_wait(&serialState->cond, &serialState->mutex); } /* A future job may error and skip our job */ @@ -932,7 +933,7 @@ static void ZSTDMT_waitForAllJobsCompleted(ZSTDMT_CCtx* mtctx) unsigned const jobID = mtctx->doneJobID & mtctx->jobIDMask; ZSTD_PTHREAD_MUTEX_LOCK(&mtctx->jobs[jobID].job_mutex); while (mtctx->jobs[jobID].consumed < mtctx->jobs[jobID].src.size) { - DEBUGLOG(5, "waiting for jobCompleted signal from job %u", mtctx->doneJobID); /* we want to block when waiting for data to flush */ + DEBUGLOG(4, "waiting for jobCompleted signal from job %u", mtctx->doneJobID); /* we want to block when waiting for data to flush */ ZSTD_pthread_cond_wait(&mtctx->jobs[jobID].job_cond, &mtctx->jobs[jobID].job_mutex); } ZSTD_pthread_mutex_unlock(&mtctx->jobs[jobID].job_mutex); @@ -1079,7 +1080,7 @@ void ZSTDMT_updateCParams_whileCompressing(ZSTDMT_CCtx* mtctx, const ZSTD_CCtx_p ZSTD_frameProgression ZSTDMT_getFrameProgression(ZSTDMT_CCtx* mtctx) { ZSTD_frameProgression fps; - DEBUGLOG(6, "ZSTDMT_getFrameProgression"); + DEBUGLOG(5, "ZSTDMT_getFrameProgression"); fps.ingested = mtctx->consumed + mtctx->inBuff.filled; fps.consumed = mtctx->consumed; fps.produced = mtctx->produced; @@ -1100,6 +1101,7 @@ ZSTD_frameProgression ZSTDMT_getFrameProgression(ZSTDMT_CCtx* mtctx) ZSTD_pthread_mutex_unlock(&mtctx->jobs[wJobID].job_mutex); } } + DEBUGLOG(5, "ZSTDMT_getFrameProgression : completed"); return fps; } @@ -1576,7 +1578,7 @@ static size_t ZSTDMT_flushProduced(ZSTDMT_CCtx* mtctx, ZSTD_outBuffer* output, u /* try to flush something */ { size_t cSize = mtctx->jobs[wJobID].cSize; /* shared */ size_t const srcConsumed = mtctx->jobs[wJobID].consumed; /* shared */ - size_t const srcSize = mtctx->jobs[wJobID].src.size; /* read-only, could be done after mutex lock, but no-declaration-after-statement */ + size_t const srcSize = mtctx->jobs[wJobID].src.size; /* read-only, could be done after mutex lock, but no-declaration-after-statement */ ZSTD_pthread_mutex_unlock(&mtctx->jobs[wJobID].job_mutex); if (ZSTD_isError(cSize)) { DEBUGLOG(5, "ZSTDMT_flushProduced: job %u : compression error detected : %s", @@ -1615,6 +1617,7 @@ static size_t ZSTDMT_flushProduced(ZSTDMT_CCtx* mtctx, ZSTD_outBuffer* output, u DEBUGLOG(5, "Job %u completed (%u bytes), moving to next one", mtctx->doneJobID, (U32)mtctx->jobs[wJobID].dstFlushed); ZSTDMT_releaseBuffer(mtctx->bufPool, mtctx->jobs[wJobID].dstBuff); + DEBUGLOG(5, "dstBuffer released") mtctx->jobs[wJobID].dstBuff = g_nullBuffer; mtctx->jobs[wJobID].cSize = 0; /* ensure this job slot is considered "not started" in future check */ mtctx->consumed += srcSize; @@ -1691,6 +1694,7 @@ static int ZSTDMT_doesOverlapWindow(buffer_t buffer, ZSTD_window_t window) range_t extDict; range_t prefix; + DEBUGLOG(5, "ZSTDMT_doesOverlapWindow"); extDict.start = window.dictBase + window.lowLimit; extDict.size = window.dictLimit - window.lowLimit; @@ -1711,12 +1715,13 @@ static void ZSTDMT_waitForLdmComplete(ZSTDMT_CCtx* mtctx, buffer_t buffer) { if (mtctx->params.ldmParams.enableLdm) { ZSTD_pthread_mutex_t* mutex = &mtctx->serial.ldmWindowMutex; + DEBUGLOG(5, "ZSTDMT_waitForLdmComplete"); DEBUGLOG(5, "source [0x%zx, 0x%zx)", (size_t)buffer.start, (size_t)buffer.start + buffer.capacity); ZSTD_PTHREAD_MUTEX_LOCK(mutex); while (ZSTDMT_doesOverlapWindow(buffer, mtctx->serial.ldmWindow)) { - DEBUGLOG(6, "Waiting for LDM to finish..."); + DEBUGLOG(5, "Waiting for LDM to finish..."); ZSTD_pthread_cond_wait(&mtctx->serial.ldmWindowCond, mutex); } DEBUGLOG(6, "Done waiting for LDM to finish"); @@ -1736,6 +1741,7 @@ static int ZSTDMT_tryGetInputRange(ZSTDMT_CCtx* mtctx) size_t const target = mtctx->targetSectionSize; buffer_t buffer; + DEBUGLOG(5, "ZSTDMT_tryGetInputRange"); assert(mtctx->inBuff.buffer.start == NULL); assert(mtctx->roundBuff.capacity >= target); @@ -1749,7 +1755,7 @@ static int ZSTDMT_tryGetInputRange(ZSTDMT_CCtx* mtctx) buffer.start = start; buffer.capacity = prefixSize; if (ZSTDMT_isOverlapped(buffer, inUse)) { - DEBUGLOG(6, "Waiting for buffer..."); + DEBUGLOG(5, "Waiting for buffer..."); return 0; } ZSTDMT_waitForLdmComplete(mtctx, buffer); @@ -1761,7 +1767,7 @@ static int ZSTDMT_tryGetInputRange(ZSTDMT_CCtx* mtctx) buffer.capacity = target; if (ZSTDMT_isOverlapped(buffer, inUse)) { - DEBUGLOG(6, "Waiting for buffer..."); + DEBUGLOG(5, "Waiting for buffer..."); return 0; } assert(!ZSTDMT_isOverlapped(buffer, mtctx->inBuff.prefix)); @@ -1834,8 +1840,10 @@ size_t ZSTDMT_compressStream_generic(ZSTDMT_CCtx* mtctx, /* It is only possible for this operation to fail if there are * still compression jobs ongoing. */ + DEBUGLOG(5, "ZSTDMT_tryGetInputRange failed") assert(mtctx->doneJobID != mtctx->nextJobID); - } + } else + DEBUGLOG(5, "ZSTDMT_tryGetInputRange completed successfully : mtctx->inBuff.buffer.start = %p", mtctx->inBuff.buffer.start); } if (mtctx->inBuff.buffer.start != NULL) { size_t const toLoad = MIN(input->size - input->pos, mtctx->targetSectionSize - mtctx->inBuff.filled); @@ -1863,6 +1871,7 @@ size_t ZSTDMT_compressStream_generic(ZSTDMT_CCtx* mtctx, /* check for potential compressed data ready to be flushed */ { size_t const remainingToFlush = ZSTDMT_flushProduced(mtctx, output, !forwardInputProgress, endOp); /* block if there was no forward input progress */ if (input->pos < input->size) return MAX(remainingToFlush, 1); /* input not consumed : do not end flush yet */ + DEBUGLOG(5, "end of ZSTDMT_compressStream_generic: remainingToFlush = %u", (U32)remainingToFlush); return remainingToFlush; } } diff --git a/programs/fileio.c b/programs/fileio.c index c12126168..89ee524b3 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -226,6 +226,8 @@ void FIO_setOverlapLog(unsigned overlapLog){ DISPLAYLEVEL(2, "Setting overlapLog is useless in single-thread mode \n"); g_overlapLog = overlapLog; } +static U32 g_adaptiveMode = 0; +void FIO_setAdaptiveMode(unsigned adapt) { g_adaptiveMode = adapt; } static U32 g_ldmFlag = 0; void FIO_setLdmFlag(unsigned ldmFlag) { g_ldmFlag = (ldmFlag>0); @@ -738,12 +740,12 @@ FIO_compressZstdFrame(const cRess_t* ressPtr, U64 compressedfilesize = 0; ZSTD_EndDirective directive = ZSTD_e_continue; + /* stats */ typedef enum { noChange, slower, faster } speedChange_e; speedChange_e speedChange = noChange; + unsigned inputPresented = 0; unsigned inputBlocked = 0; unsigned lastJobID = 0; - unsigned long long lastProduced = 0; - unsigned long long lastFlushedSize = 0; DISPLAYLEVEL(6, "compression using zstd format \n"); @@ -774,6 +776,7 @@ FIO_compressZstdFrame(const cRess_t* ressPtr, CHECK_V(stillToFlush, ZSTD_compress_generic(ress.cctx, &outBuff, &inBuff, directive)); /* count stats */ + inputPresented++; if (oldIPos == inBuff.pos) inputBlocked++; /* Write compressed stream */ @@ -792,41 +795,74 @@ FIO_compressZstdFrame(const cRess_t* ressPtr, double const cShare = (double)zfp.produced / (zfp.consumed + !zfp.consumed/*avoid div0*/) * 100; /* check output speed */ - if (zfp.currentJobID > 0) { - unsigned long long newlyProduced = zfp.produced - lastProduced; + if (zfp.currentJobID > 1) { + static ZSTD_frameProgression cpszfp = { 0, 0, 0, 0 }; + static unsigned long long lastFlushedSize = 0; + + unsigned long long newlyProduced = zfp.produced - cpszfp.produced; unsigned long long newlyFlushed = compressedfilesize - lastFlushedSize; - assert(zfp.produced >= lastProduced); - if (newlyProduced == 0) { - DISPLAYLEVEL(6, "no more data compression generation => buffers are full, compression waiting => output (or input) too slow \n") + assert(zfp.produced >= cpszfp.produced); + + if ( (zfp.ingested == cpszfp.ingested) + && (zfp.consumed == cpszfp.consumed) ) { + DISPLAYLEVEL(6, "no data read nor consumed : buffers are full (?) or compression is slow + input has reached its limit. If buffers full : output is too slow => slow down \n") speedChange = slower; } if ( (newlyProduced > (newlyFlushed * 9 / 8)) && (stillToFlush > ZSTD_BLOCKSIZE_MAX) ) { - DISPLAYLEVEL(6, "production faster than flushing (%llu > %llu) \n", newlyProduced, newlyFlushed); + DISPLAYLEVEL(6, "production faster than flushing (%llu > %llu) but there is still %u bytes to flush => slow down \n", newlyProduced, newlyFlushed, (U32)stillToFlush); speedChange = slower; } - lastProduced = zfp.produced; + cpszfp = zfp; lastFlushedSize = compressedfilesize; } - /* course correct only if there is at least one job completed */ + /* course correct only if there is at least one new job completed */ if (zfp.currentJobID > lastJobID) { DISPLAYLEVEL(6, "compression level adaptation check \n") /* check input speed */ if (zfp.currentJobID > g_nbWorkers+1) { /* warm up period, to fill all workers */ - if (inputBlocked <= 1) { /* small tolerance */ + if (inputBlocked <= 0) { DISPLAYLEVEL(6, "input is never blocked => input is too slow \n"); speedChange = slower; + } else if (speedChange == noChange) { + static ZSTD_frameProgression csuzfp = { 0, 0, 0, 0 }; + static unsigned long long lastFlushedSize = 0; + unsigned long long newlyIngested = zfp.ingested - csuzfp.ingested; + unsigned long long newlyConsumed = zfp.consumed - csuzfp.consumed; + unsigned long long newlyProduced = zfp.produced - csuzfp.produced; + unsigned long long newlyFlushed = compressedfilesize - lastFlushedSize; + csuzfp = zfp; + lastFlushedSize = compressedfilesize; + assert(inputPresented > 0); + if ( (inputBlocked > inputPresented / 8) /* input is waiting often, because input buffers is full : compression or output too slow */ + && (newlyFlushed * 17 / 16 > newlyProduced) /* flush everything that is produced */ + && (newlyIngested * 17 / 16 > newlyConsumed) /* can't keep up with input speed */ + ) { + DISPLAYLEVEL(6, "recommend faster as in(%llu) >= (%llu)comp(%llu) <= out(%llu) \n", + newlyIngested, newlyConsumed, newlyProduced, newlyFlushed); + speedChange = faster; + } } inputBlocked = 0; + inputPresented = 0; } - if (speedChange == slower) { - DISPLAYLEVEL(6, "slower speed , higher compression \n") - compressionLevel ++; - ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_compressionLevel, (unsigned)compressionLevel); + if (g_adaptiveMode) { + if (speedChange == slower) { + DISPLAYLEVEL(6, "slower speed , higher compression \n") + compressionLevel ++; + compressionLevel += (compressionLevel == 0); /* skip 0 */ + ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_compressionLevel, (unsigned)compressionLevel); + } + if (speedChange == faster) { + DISPLAYLEVEL(6, "slower speed , higher compression \n") + compressionLevel --; + compressionLevel -= (compressionLevel == 0); /* skip 0 */ + ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_compressionLevel, (unsigned)compressionLevel); + } speedChange = noChange; } lastJobID = zfp.currentJobID; diff --git a/programs/fileio.h b/programs/fileio.h index 69c83f71d..f4946c78a 100644 --- a/programs/fileio.h +++ b/programs/fileio.h @@ -57,6 +57,7 @@ void FIO_setMemLimit(unsigned memLimit); void FIO_setNbWorkers(unsigned nbWorkers); void FIO_setBlockSize(unsigned blockSize); void FIO_setOverlapLog(unsigned overlapLog); +void FIO_setAdaptiveMode(unsigned adapt); void FIO_setLdmFlag(unsigned ldmFlag); void FIO_setLdmHashLog(unsigned ldmHashLog); void FIO_setLdmMinMatch(unsigned ldmMinMatch); diff --git a/programs/zstd.1.md b/programs/zstd.1.md index 055c5c244..b71d5d5bf 100644 --- a/programs/zstd.1.md +++ b/programs/zstd.1.md @@ -102,6 +102,13 @@ the last one takes effect. * `-#`: `#` compression level \[1-19] (default: 3) +* `--fast[=#]`: + switch to ultra-fast compression levels. + If `=#` is not present, it defaults to `1`. + The higher the value, the faster the compression speed, + at the cost of some compression ratio. + This setting overwrites compression level if one was set previously. + Similarly, if a compression level is set after `--fast`, it overrides it. * `--ultra`: unlocks high compression levels 20+ (maximum 22), using a lot more memory. Note that decompression will also require more memory when using these levels. @@ -115,25 +122,23 @@ the last one takes effect. Note: If `windowLog` is set to larger than 27, `--long=windowLog` or `--memory=windowSize` needs to be passed to the decompressor. -* `--fast[=#]`: - switch to ultra-fast compression levels. - If `=#` is not present, it defaults to `1`. - The higher the value, the faster the compression speed, - at the cost of some compression ratio. - This setting overwrites compression level if one was set previously. - Similarly, if a compression level is set after `--fast`, it overrides it. - * `-T#`, `--threads=#`: Compress using `#` working threads (default: 1). If `#` is 0, attempt to detect and use the number of physical CPU cores. In all cases, the nb of threads is capped to ZSTDMT_NBTHREADS_MAX==200. This modifier does nothing if `zstd` is compiled without multithread support. * `--single-thread`: - Does not spawn a thread for compression, use caller thread instead. - This is the only available mode when multithread support is disabled. - In this mode, compression is serialized with I/O. + Does not spawn a thread for compression, use a single thread for both I/O and compression. + In this mode, compression is serialized with I/O, which is slightly slower. (This is different from `-T1`, which spawns 1 compression thread in parallel of I/O). - Single-thread mode also features lower memory usage. + This mode is the only one available when multithread support is disabled. + Single-thread mode features lower memory usage. + Final compressed result is slightly different from `-T1`. +* `--adapt` : + `zstd` will dynamically adapt compression level to perceived I/O conditions. + The current compression level can be observed live by using command `-v`. + Works with multi-threading and `--long` mode. + Does not work with `--single-thread`. * `-D file`: use `file` as Dictionary to compress or decompress FILE(s) * `--no-dictID`: diff --git a/programs/zstdcli.c b/programs/zstdcli.c index d5a2216d6..2e54b3b0f 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -135,6 +135,7 @@ static int usage_advanced(const char* programName) #ifndef ZSTD_NOCOMPRESS DISPLAY( "--ultra : enable levels beyond %i, up to %i (requires more memory)\n", ZSTDCLI_CLEVEL_MAX, ZSTD_maxCLevel()); DISPLAY( "--long[=#]: enable long distance matching with given window log (default: %u)\n", g_defaultMaxWindowLog); + DISPLAY( "--adapt : automatically adapt compression level to I/O conditions \n"); DISPLAY( "--fast[=#]: switch to ultra fast compression level (default: %u)\n", 1); #ifdef ZSTD_MULTITHREAD DISPLAY( " -T# : spawns # compression threads (default: 1, 0==# cores) \n"); @@ -395,6 +396,7 @@ int main(int argCount, const char* argv[]) ldmFlag = 0, main_pause = 0, nbWorkers = 0, + adapt = 0, nextArgumentIsOutFileName = 0, nextArgumentIsMaxDict = 0, nextArgumentIsDictID = 0, @@ -511,6 +513,7 @@ int main(int argCount, const char* argv[]) if (!strcmp(argument, "--keep")) { FIO_setRemoveSrcFile(0); continue; } if (!strcmp(argument, "--rm")) { FIO_setRemoveSrcFile(1); continue; } if (!strcmp(argument, "--priority=rt")) { setRealTimePrio = 1; continue; } + if (!strcmp(argument, "--adapt")) { adapt = 1; continue; } if (!strcmp(argument, "--single-thread")) { nbWorkers = 0; singleThread = 1; continue; } if (!strcmp(argument, "--format=zstd")) { suffix = ZSTD_EXTENSION; FIO_setCompressionType(FIO_zstdCompression); continue; } #ifdef ZSTD_GZCOMPRESS @@ -935,17 +938,14 @@ int main(int argCount, const char* argv[]) #ifndef ZSTD_NOCOMPRESS FIO_setNbWorkers(nbWorkers); FIO_setBlockSize((U32)blockSize); + if (g_overlapLog!=OVERLAP_LOG_DEFAULT) FIO_setOverlapLog(g_overlapLog); FIO_setLdmFlag(ldmFlag); FIO_setLdmHashLog(g_ldmHashLog); FIO_setLdmMinMatch(g_ldmMinMatch); - if (g_ldmBucketSizeLog != LDM_PARAM_DEFAULT) { - FIO_setLdmBucketSizeLog(g_ldmBucketSizeLog); - } - if (g_ldmHashEveryLog != LDM_PARAM_DEFAULT) { - FIO_setLdmHashEveryLog(g_ldmHashEveryLog); - } + if (g_ldmBucketSizeLog != LDM_PARAM_DEFAULT) FIO_setLdmBucketSizeLog(g_ldmBucketSizeLog); + if (g_ldmHashEveryLog != LDM_PARAM_DEFAULT) FIO_setLdmHashEveryLog(g_ldmHashEveryLog); + FIO_setAdaptiveMode(adapt); - if (g_overlapLog!=OVERLAP_LOG_DEFAULT) FIO_setOverlapLog(g_overlapLog); if ((filenameIdx==1) && outFileName) operationResult = FIO_compressFilename(outFileName, filenameTable[0], dictFileName, cLevel, &compressionParams); else From f3aa510738baa7d4656dfbe7998f5c09db4c7bf7 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Mon, 13 Aug 2018 11:38:55 -0700 Subject: [PATCH 136/372] rateLimiter does not "catch up" when input speed is slow --- programs/fileio.c | 8 ++++++-- programs/zstd.1.md | 1 + tests/rateLimiter.py | 11 ++++++++--- 3 files changed, 15 insertions(+), 5 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index 89ee524b3..c3eddad49 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -837,9 +837,13 @@ FIO_compressZstdFrame(const cRess_t* ressPtr, csuzfp = zfp; lastFlushedSize = compressedfilesize; assert(inputPresented > 0); + DISPLAYLEVEL(6, "input blocked %u/%u(%.2f) - ingested:%u vs %u:consumed - flushed:%u vs %u:produced \n", + inputBlocked, inputPresented, (double)inputBlocked/inputPresented*100, + (U32)newlyIngested, (U32)newlyConsumed, + (U32)newlyFlushed, (U32)newlyProduced); if ( (inputBlocked > inputPresented / 8) /* input is waiting often, because input buffers is full : compression or output too slow */ - && (newlyFlushed * 17 / 16 > newlyProduced) /* flush everything that is produced */ - && (newlyIngested * 17 / 16 > newlyConsumed) /* can't keep up with input speed */ + && (newlyFlushed * 33 / 32 > newlyProduced) /* flush everything that is produced */ + && (newlyIngested * 33 / 32 > newlyConsumed) /* input speed as fast or faster than compression speed */ ) { DISPLAYLEVEL(6, "recommend faster as in(%llu) >= (%llu)comp(%llu) <= out(%llu) \n", newlyIngested, newlyConsumed, newlyProduced, newlyFlushed); diff --git a/programs/zstd.1.md b/programs/zstd.1.md index b71d5d5bf..5f3701864 100644 --- a/programs/zstd.1.md +++ b/programs/zstd.1.md @@ -139,6 +139,7 @@ the last one takes effect. The current compression level can be observed live by using command `-v`. Works with multi-threading and `--long` mode. Does not work with `--single-thread`. + Due to the chaotic nature of dynamic adaptation, compressed result is not reproducible. * `-D file`: use `file` as Dictionary to compress or decompress FILE(s) * `--no-dictID`: diff --git a/tests/rateLimiter.py b/tests/rateLimiter.py index 134ef8971..15222e016 100755 --- a/tests/rateLimiter.py +++ b/tests/rateLimiter.py @@ -19,7 +19,7 @@ import time MB = 1024 * 1024 rate = float(sys.argv[1]) * MB -rate *= 1.25 # compensation for excluding write time (experimentally determined) +rate *= 1.4 # compensation for excluding i/o time (experimentally determined) start = time.time() total_read = 0 @@ -29,9 +29,14 @@ while len(buf): to_read = max(int(rate * (now - start) - total_read), 1) max_buf_size = 1 * MB to_read = min(to_read, max_buf_size) + + read_start = time.time() buf = sys.stdin.buffer.read(to_read) - write_start = time.time() + + write_start = read_end = time.time() sys.stdout.buffer.write(buf) write_end = time.time() - start += write_end - write_start # exclude write delay + + wait_time = max(read_end - read_start, write_end - write_start) + start += wait_time # exclude delay of the slowest total_read += len(buf) From e11f91b0399a7379bbaf96728ebedc61cbc0b34e Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Mon, 13 Aug 2018 11:48:25 -0700 Subject: [PATCH 137/372] remove error message for Ctrl+C --- tests/rateLimiter.py | 34 ++++++++++++++++++++-------------- 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/tests/rateLimiter.py b/tests/rateLimiter.py index 15222e016..d69523ca3 100755 --- a/tests/rateLimiter.py +++ b/tests/rateLimiter.py @@ -23,20 +23,26 @@ rate *= 1.4 # compensation for excluding i/o time (experimentally determined) start = time.time() total_read = 0 -buf = " " -while len(buf): - now = time.time() - to_read = max(int(rate * (now - start) - total_read), 1) - max_buf_size = 1 * MB - to_read = min(to_read, max_buf_size) +sys.stderr.close() # remove error message, for Ctrl+C - read_start = time.time() - buf = sys.stdin.buffer.read(to_read) +try: + buf = " " + while len(buf): + now = time.time() + to_read = max(int(rate * (now - start) - total_read), 1) + max_buf_size = 1 * MB + to_read = min(to_read, max_buf_size) - write_start = read_end = time.time() - sys.stdout.buffer.write(buf) - write_end = time.time() + read_start = time.time() + buf = sys.stdin.buffer.read(to_read) - wait_time = max(read_end - read_start, write_end - write_start) - start += wait_time # exclude delay of the slowest - total_read += len(buf) + write_start = read_end = time.time() + sys.stdout.buffer.write(buf) + write_end = time.time() + + wait_time = max(read_end - read_start, write_end - write_start) + start += wait_time # exclude delay of the slowest + total_read += len(buf) + +except (KeyboardInterrupt, BrokenPipeError) as e: + pass From 09c9cf3f51c85fc961dea176ded938aa1e685760 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Mon, 13 Aug 2018 12:13:47 -0700 Subject: [PATCH 138/372] simplified rateLimiter resists better to changing in/out conditions limits risks of "catching up" --- tests/rateLimiter.py | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/tests/rateLimiter.py b/tests/rateLimiter.py index d69523ca3..da0baf014 100755 --- a/tests/rateLimiter.py +++ b/tests/rateLimiter.py @@ -19,30 +19,22 @@ import time MB = 1024 * 1024 rate = float(sys.argv[1]) * MB -rate *= 1.4 # compensation for excluding i/o time (experimentally determined) start = time.time() total_read = 0 -sys.stderr.close() # remove error message, for Ctrl+C +# sys.stderr.close() # remove error message, for Ctrl+C try: buf = " " while len(buf): now = time.time() - to_read = max(int(rate * (now - start) - total_read), 1) + to_read = max(int(rate * (now - start)), 1) max_buf_size = 1 * MB to_read = min(to_read, max_buf_size) + start = now - read_start = time.time() buf = sys.stdin.buffer.read(to_read) - - write_start = read_end = time.time() sys.stdout.buffer.write(buf) - write_end = time.time() - - wait_time = max(read_end - read_start, write_end - write_start) - start += wait_time # exclude delay of the slowest - total_read += len(buf) except (KeyboardInterrupt, BrokenPipeError) as e: pass From 33f7709c711ae1834ad7c48e02b6b1909584aa9d Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Mon, 13 Aug 2018 13:02:03 -0700 Subject: [PATCH 139/372] fileio: changed parameter type from ptr to plain structure safer : this parameter is read-only, we don't want original structure to be modified --- doc/zstd_manual.html | 1 + programs/fileio.c | 21 +++++++++++---------- programs/fileio.h | 4 ++-- programs/zstdcli.c | 4 ++-- 4 files changed, 16 insertions(+), 14 deletions(-) diff --git a/doc/zstd_manual.html b/doc/zstd_manual.html index bd792008b..48b9ca7f2 100644 --- a/doc/zstd_manual.html +++ b/doc/zstd_manual.html @@ -599,6 +599,7 @@ size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs, const ZSTD_CDict* unsigned long long ingested; unsigned long long consumed; unsigned long long produced; + unsigned currentJobID; } ZSTD_frameProgression;

Advanced Streaming decompression functions

typedef enum { DStream_p_maxWindowSize } ZSTD_DStreamParameter_e;
diff --git a/programs/fileio.c b/programs/fileio.c
index c3eddad49..8803fff0f 100644
--- a/programs/fileio.c
+++ b/programs/fileio.c
@@ -406,7 +406,7 @@ typedef struct {
 
 static cRess_t FIO_createCResources(const char* dictFileName, int cLevel,
                                     U64 srcSize,
-                                    ZSTD_compressionParameters* comprParams) {
+                                    ZSTD_compressionParameters comprParams) {
     cRess_t ress;
     memset(&ress, 0, sizeof(ress));
 
@@ -443,13 +443,13 @@ static cRess_t FIO_createCResources(const char* dictFileName, int cLevel,
             CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_ldmHashEveryLog, g_ldmHashEveryLog) );
         }
         /* compression parameters */
-        CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_windowLog, comprParams->windowLog) );
-        CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_chainLog, comprParams->chainLog) );
-        CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_hashLog, comprParams->hashLog) );
-        CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_searchLog, comprParams->searchLog) );
-        CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_minMatch, comprParams->searchLength) );
-        CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_targetLength, comprParams->targetLength) );
-        CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_compressionStrategy, (U32)comprParams->strategy) );
+        CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_windowLog, comprParams.windowLog) );
+        CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_chainLog, comprParams.chainLog) );
+        CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_hashLog, comprParams.hashLog) );
+        CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_searchLog, comprParams.searchLog) );
+        CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_minMatch, comprParams.searchLength) );
+        CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_targetLength, comprParams.targetLength) );
+        CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_compressionStrategy, (U32)comprParams.strategy) );
         /* multi-threading */
 #ifdef ZSTD_MULTITHREAD
         DISPLAYLEVEL(5,"set nb workers = %u \n", g_nbWorkers);
@@ -1048,7 +1048,8 @@ static int FIO_compressFilename_dstFile(cRess_t ress,
 
 
 int FIO_compressFilename(const char* dstFileName, const char* srcFileName,
-                         const char* dictFileName, int compressionLevel, ZSTD_compressionParameters* comprParams)
+                         const char* dictFileName, int compressionLevel,
+                         ZSTD_compressionParameters comprParams)
 {
     clock_t const start = clock();
     U64 const fileSize = UTIL_getFileSize(srcFileName);
@@ -1068,7 +1069,7 @@ int FIO_compressFilename(const char* dstFileName, const char* srcFileName,
 int FIO_compressMultipleFilenames(const char** inFileNamesTable, unsigned nbFiles,
                                   const char* outFileName, const char* suffix,
                                   const char* dictFileName, int compressionLevel,
-                                  ZSTD_compressionParameters* comprParams)
+                                  ZSTD_compressionParameters comprParams)
 {
     int missed_files = 0;
     size_t dfnSize = FNSPACE;
diff --git a/programs/fileio.h b/programs/fileio.h
index f4946c78a..9c1f357a6 100644
--- a/programs/fileio.h
+++ b/programs/fileio.h
@@ -71,7 +71,7 @@ void FIO_setLdmHashEveryLog(unsigned ldmHashEveryLog);
 /** FIO_compressFilename() :
     @return : 0 == ok;  1 == pb with src file. */
 int FIO_compressFilename (const char* outfilename, const char* infilename, const char* dictFileName,
-                          int compressionLevel, ZSTD_compressionParameters* comprParams);
+                          int compressionLevel, ZSTD_compressionParameters comprParams);
 
 /** FIO_decompressFilename() :
     @return : 0 == ok;  1 == pb with src file. */
@@ -87,7 +87,7 @@ int FIO_listMultipleFiles(unsigned numFiles, const char** filenameTable, int dis
 int FIO_compressMultipleFilenames(const char** srcNamesTable, unsigned nbFiles,
                                   const char* outFileName, const char* suffix,
                                   const char* dictFileName, int compressionLevel,
-                                  ZSTD_compressionParameters* comprParams);
+                                  ZSTD_compressionParameters comprParams);
 
 /** FIO_decompressMultipleFilenames() :
     @return : nb of missing or skipped files */
diff --git a/programs/zstdcli.c b/programs/zstdcli.c
index 2e54b3b0f..d1bb5ed25 100644
--- a/programs/zstdcli.c
+++ b/programs/zstdcli.c
@@ -947,9 +947,9 @@ int main(int argCount, const char* argv[])
         FIO_setAdaptiveMode(adapt);
 
         if ((filenameIdx==1) && outFileName)
-          operationResult = FIO_compressFilename(outFileName, filenameTable[0], dictFileName, cLevel, &compressionParams);
+          operationResult = FIO_compressFilename(outFileName, filenameTable[0], dictFileName, cLevel, compressionParams);
         else
-          operationResult = FIO_compressMultipleFilenames(filenameTable, filenameIdx, outFileName, suffix, dictFileName, cLevel, &compressionParams);
+          operationResult = FIO_compressMultipleFilenames(filenameTable, filenameIdx, outFileName, suffix, dictFileName, cLevel, compressionParams);
 #else
         (void)suffix;
         DISPLAY("Compression not supported\n");

From 0853f8604474d7c519f4e98fd338bd57c5812006 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Mon, 13 Aug 2018 13:10:42 -0700
Subject: [PATCH 140/372] adaptive mode uses default window size of 8 MB

---
 programs/fileio.c  | 4 ++++
 programs/zstd.1.md | 5 +++--
 2 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/programs/fileio.c b/programs/fileio.c
index 8803fff0f..b29936191 100644
--- a/programs/fileio.c
+++ b/programs/fileio.c
@@ -71,6 +71,7 @@
 #define MB *(1<<20)
 #define GB *(1U<<30)
 
+#define ADAPT_WINDOWLOG_DEFAULT 23   /* 8 MB */
 #define DICTSIZE_MAX (32 MB)   /* protection against large input (attack scenario) */
 
 #define FNSPACE 30
@@ -427,6 +428,9 @@ static cRess_t FIO_createCResources(const char* dictFileName, int cLevel,
         if (dictFileName && (dictBuffer==NULL))
             EXM_THROW(32, "allocation error : can't create dictBuffer");
 
+        if (g_adaptiveMode && !g_ldmFlag && !comprParams.windowLog)
+            comprParams.windowLog = ADAPT_WINDOWLOG_DEFAULT;
+
         CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_contentSizeFlag, 1) );  /* always enable content size when available (note: supposed to be default) */
         CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_dictIDFlag, g_dictIDFlag) );
         CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_checksumFlag, g_checksumFlag) );
diff --git a/programs/zstd.1.md b/programs/zstd.1.md
index 5f3701864..b662758ac 100644
--- a/programs/zstd.1.md
+++ b/programs/zstd.1.md
@@ -136,10 +136,11 @@ the last one takes effect.
     Final compressed result is slightly different from `-T1`.
 * `--adapt` :
     `zstd` will dynamically adapt compression level to perceived I/O conditions.
-    The current compression level can be observed live by using command `-v`.
+    Compression level adaptation can be observed live by using command `-v`.
     Works with multi-threading and `--long` mode.
-    Does not work with `--single-thread`.
+    When not specified otherwise, uses a window size of 8 MB.
     Due to the chaotic nature of dynamic adaptation, compressed result is not reproducible.
+    Adaptation does not work with `--single-thread`.
 * `-D file`:
     use `file` as Dictionary to compress or decompress FILE(s)
 * `--no-dictID`:

From 0cc75d6ee02efe9da7901848ce588a7f9f61b4f9 Mon Sep 17 00:00:00 2001
From: George Lu 
Date: Mon, 13 Aug 2018 13:56:18 -0700
Subject: [PATCH 141/372] Default lvl 1

MB to 2^20
---
 tests/paramgrill.c | 27 ++++++++++++++-------------
 1 file changed, 14 insertions(+), 13 deletions(-)

diff --git a/tests/paramgrill.c b/tests/paramgrill.c
index bcb6bcae3..d7165021d 100644
--- a/tests/paramgrill.c
+++ b/tests/paramgrill.c
@@ -65,6 +65,7 @@ static const int g_maxNbVariations = 64;
 #define MIN(a,b)   ( (a) < (b) ? (a) : (b) )
 #define MAX(a,b)   ( (a) > (b) ? (a) : (b) )
 #define CUSTOM_LEVEL 99
+#define BASE_CLEVEL 1
 
 /* indices for each of the variables */
 typedef enum {
@@ -302,7 +303,7 @@ BMK_benchParam1(BMK_result_t* resultPtr,
                const void* srcBuffer, size_t srcSize,
                const ZSTD_compressionParameters cParams) {
 
-    BMK_return_t res = BMK_benchMem(srcBuffer,srcSize, &srcSize, 1, 0, &cParams, NULL, 0, 0, "File");
+    BMK_return_t res = BMK_benchMem(srcBuffer,srcSize, &srcSize, 1, BASE_CLEVEL, &cParams, NULL, 0, 0, "File");
     *resultPtr = res.result;
     return res.error;
 }
@@ -792,16 +793,16 @@ static int BMK_seed(winnerInfo_t* winners, const ZSTD_compressionParameters para
                 /* too large compression speed difference for the compression benefit */
                 if (W_ratio > O_ratio)
                 DISPLAY ("Compression Speed : %5.3f @ %4.1f MB/s  vs  %5.3f @ %4.1f MB/s   : not enough for level %i\n",
-                         W_ratio, (double)testResult.cSpeed / 1000000,
-                         O_ratio, (double)winners[cLevel].result.cSpeed / 1000000.,   cLevel);
+                         W_ratio, (double)testResult.cSpeed / (1 MB),
+                         O_ratio, (double)winners[cLevel].result.cSpeed / (1 MB),   cLevel);
                 continue;
             }
             if (W_DSpeed_note   < O_DSpeed_note  ) {
                 /* too large decompression speed difference for the compression benefit */
                 if (W_ratio > O_ratio)
                 DISPLAY ("Decompression Speed : %5.3f @ %4.1f MB/s  vs  %5.3f @ %4.1f MB/s   : not enough for level %i\n",
-                         W_ratio, (double)testResult.dSpeed / 1000000.,
-                         O_ratio, (double)winners[cLevel].result.dSpeed / 1000000.,   cLevel);
+                         W_ratio, (double)testResult.dSpeed / (1 MB),
+                         O_ratio, (double)winners[cLevel].result.dSpeed / (1 MB),   cLevel);
                 continue;
             }
 
@@ -1193,7 +1194,7 @@ static void BMK_benchOnce(const void* srcBuffer, size_t srcSize)
     g_params = ZSTD_adjustCParams(g_params, srcSize, 0);
     BMK_benchParam1(&testResult, srcBuffer, srcSize, g_params);
     DISPLAY("Compression Ratio: %.3f  Compress Speed: %.1f MB/s Decompress Speed: %.1f MB/s\n", (double)srcSize / testResult.cSize, 
-        (double)testResult.cSpeed / 1000000, (double)testResult.dSpeed / 1000000);
+        (double)testResult.cSpeed / (1 MB), (double)testResult.dSpeed / (1 MB));
     return;
 }
 
@@ -1211,7 +1212,7 @@ static void BMK_benchFullTable(const void* srcBuffer, size_t srcSize)
     if (f==NULL) { DISPLAY("error opening %s \n", rfName); exit(1); }
 
     if (g_target) {
-        BMK_init_level_constraints(g_target*1000000);
+        BMK_init_level_constraints(g_target*(1 MB));
     } else {
         /* baseline config for level 1 */
         ZSTD_compressionParameters const l1params = ZSTD_getCParams(1, blockSize, 0);
@@ -1256,7 +1257,7 @@ static void BMK_benchMemInit(const void* srcBuffer, size_t srcSize)
 static int benchSample(void)
 {
     const char* const name = "Sample 10MB";
-    size_t const benchedSize = 10000000;
+    size_t const benchedSize = (10 MB);
 
     void* origBuff = malloc(benchedSize);
     if (!origBuff) { perror("not enough memory"); return 12; }
@@ -1354,7 +1355,7 @@ static int allBench(BMK_result_t* resultPtr,
     double winnerRS;
 
     /* initial benchmarking, gives exact ratio and memory, warms up future runs */
-    benchres = BMK_benchMemInvertible(buf, ctx, 0, &cParams, BMK_both, BMK_iterMode, 1);
+    benchres = BMK_benchMemInvertible(buf, ctx, BASE_CLEVEL, &cParams, BMK_both, BMK_iterMode, 1);
 
     winnerRS = resultScore(*winnerResult, buf.srcSize, target);
     DEBUGOUTPUT("WinnerScore: %f\n ", winnerRS);
@@ -1389,14 +1390,14 @@ static int allBench(BMK_result_t* resultPtr,
 
     /* second run, if first run is too short, gives approximate cSpeed + dSpeed */
     if(loopDurationC < TIMELOOP_NANOSEC / 10) {
-        BMK_return_t benchres2 = BMK_benchMemInvertible(buf, ctx, 0, &cParams, BMK_compressOnly, BMK_iterMode, 1);
+        BMK_return_t benchres2 = BMK_benchMemInvertible(buf, ctx, BASE_CLEVEL, &cParams, BMK_compressOnly, BMK_iterMode, 1);
         if(benchres2.error) {
             return ERROR_RESULT;
         }
         benchres = benchres2;
     }
     if(loopDurationD < TIMELOOP_NANOSEC / 10) {
-        BMK_return_t benchres2 = BMK_benchMemInvertible(buf, ctx, 0, &cParams, BMK_decodeOnly, BMK_iterMode, 1);
+        BMK_return_t benchres2 = BMK_benchMemInvertible(buf, ctx, BASE_CLEVEL, &cParams, BMK_decodeOnly, BMK_iterMode, 1);
         if(benchres2.error) {
             return ERROR_RESULT;
         }
@@ -1419,7 +1420,7 @@ static int allBench(BMK_result_t* resultPtr,
 
     /* Final full run if estimates are unclear */
     if(loopDurationC < TIMELOOP_NANOSEC) {
-        BMK_return_t benchres2 = BMK_benchMemInvertible(buf, ctx, 0, &cParams, BMK_compressOnly, BMK_timeMode, 1);
+        BMK_return_t benchres2 = BMK_benchMemInvertible(buf, ctx, BASE_CLEVEL, &cParams, BMK_compressOnly, BMK_timeMode, 1);
         if(benchres2.error) {
             return ERROR_RESULT;
         }
@@ -1427,7 +1428,7 @@ static int allBench(BMK_result_t* resultPtr,
     } 
 
     if(loopDurationD < TIMELOOP_NANOSEC) {
-        BMK_return_t benchres2 = BMK_benchMemInvertible(buf, ctx, 0, &cParams, BMK_decodeOnly, BMK_timeMode, 1);
+        BMK_return_t benchres2 = BMK_benchMemInvertible(buf, ctx, BASE_CLEVEL, &cParams, BMK_decodeOnly, BMK_timeMode, 1);
         if(benchres2.error) {
             return ERROR_RESULT;
         }

From 486e586eed5d30f6c4ec079bb403d6c15004bb9d Mon Sep 17 00:00:00 2001
From: George Lu 
Date: Mon, 13 Aug 2018 16:13:46 -0700
Subject: [PATCH 142/372] Revert "Default lvl 1"

This reverts commit 0cc75d6ee02efe9da7901848ce588a7f9f61b4f9.
---
 tests/paramgrill.c | 27 +++++++++++++--------------
 1 file changed, 13 insertions(+), 14 deletions(-)

diff --git a/tests/paramgrill.c b/tests/paramgrill.c
index d7165021d..bcb6bcae3 100644
--- a/tests/paramgrill.c
+++ b/tests/paramgrill.c
@@ -65,7 +65,6 @@ static const int g_maxNbVariations = 64;
 #define MIN(a,b)   ( (a) < (b) ? (a) : (b) )
 #define MAX(a,b)   ( (a) > (b) ? (a) : (b) )
 #define CUSTOM_LEVEL 99
-#define BASE_CLEVEL 1
 
 /* indices for each of the variables */
 typedef enum {
@@ -303,7 +302,7 @@ BMK_benchParam1(BMK_result_t* resultPtr,
                const void* srcBuffer, size_t srcSize,
                const ZSTD_compressionParameters cParams) {
 
-    BMK_return_t res = BMK_benchMem(srcBuffer,srcSize, &srcSize, 1, BASE_CLEVEL, &cParams, NULL, 0, 0, "File");
+    BMK_return_t res = BMK_benchMem(srcBuffer,srcSize, &srcSize, 1, 0, &cParams, NULL, 0, 0, "File");
     *resultPtr = res.result;
     return res.error;
 }
@@ -793,16 +792,16 @@ static int BMK_seed(winnerInfo_t* winners, const ZSTD_compressionParameters para
                 /* too large compression speed difference for the compression benefit */
                 if (W_ratio > O_ratio)
                 DISPLAY ("Compression Speed : %5.3f @ %4.1f MB/s  vs  %5.3f @ %4.1f MB/s   : not enough for level %i\n",
-                         W_ratio, (double)testResult.cSpeed / (1 MB),
-                         O_ratio, (double)winners[cLevel].result.cSpeed / (1 MB),   cLevel);
+                         W_ratio, (double)testResult.cSpeed / 1000000,
+                         O_ratio, (double)winners[cLevel].result.cSpeed / 1000000.,   cLevel);
                 continue;
             }
             if (W_DSpeed_note   < O_DSpeed_note  ) {
                 /* too large decompression speed difference for the compression benefit */
                 if (W_ratio > O_ratio)
                 DISPLAY ("Decompression Speed : %5.3f @ %4.1f MB/s  vs  %5.3f @ %4.1f MB/s   : not enough for level %i\n",
-                         W_ratio, (double)testResult.dSpeed / (1 MB),
-                         O_ratio, (double)winners[cLevel].result.dSpeed / (1 MB),   cLevel);
+                         W_ratio, (double)testResult.dSpeed / 1000000.,
+                         O_ratio, (double)winners[cLevel].result.dSpeed / 1000000.,   cLevel);
                 continue;
             }
 
@@ -1194,7 +1193,7 @@ static void BMK_benchOnce(const void* srcBuffer, size_t srcSize)
     g_params = ZSTD_adjustCParams(g_params, srcSize, 0);
     BMK_benchParam1(&testResult, srcBuffer, srcSize, g_params);
     DISPLAY("Compression Ratio: %.3f  Compress Speed: %.1f MB/s Decompress Speed: %.1f MB/s\n", (double)srcSize / testResult.cSize, 
-        (double)testResult.cSpeed / (1 MB), (double)testResult.dSpeed / (1 MB));
+        (double)testResult.cSpeed / 1000000, (double)testResult.dSpeed / 1000000);
     return;
 }
 
@@ -1212,7 +1211,7 @@ static void BMK_benchFullTable(const void* srcBuffer, size_t srcSize)
     if (f==NULL) { DISPLAY("error opening %s \n", rfName); exit(1); }
 
     if (g_target) {
-        BMK_init_level_constraints(g_target*(1 MB));
+        BMK_init_level_constraints(g_target*1000000);
     } else {
         /* baseline config for level 1 */
         ZSTD_compressionParameters const l1params = ZSTD_getCParams(1, blockSize, 0);
@@ -1257,7 +1256,7 @@ static void BMK_benchMemInit(const void* srcBuffer, size_t srcSize)
 static int benchSample(void)
 {
     const char* const name = "Sample 10MB";
-    size_t const benchedSize = (10 MB);
+    size_t const benchedSize = 10000000;
 
     void* origBuff = malloc(benchedSize);
     if (!origBuff) { perror("not enough memory"); return 12; }
@@ -1355,7 +1354,7 @@ static int allBench(BMK_result_t* resultPtr,
     double winnerRS;
 
     /* initial benchmarking, gives exact ratio and memory, warms up future runs */
-    benchres = BMK_benchMemInvertible(buf, ctx, BASE_CLEVEL, &cParams, BMK_both, BMK_iterMode, 1);
+    benchres = BMK_benchMemInvertible(buf, ctx, 0, &cParams, BMK_both, BMK_iterMode, 1);
 
     winnerRS = resultScore(*winnerResult, buf.srcSize, target);
     DEBUGOUTPUT("WinnerScore: %f\n ", winnerRS);
@@ -1390,14 +1389,14 @@ static int allBench(BMK_result_t* resultPtr,
 
     /* second run, if first run is too short, gives approximate cSpeed + dSpeed */
     if(loopDurationC < TIMELOOP_NANOSEC / 10) {
-        BMK_return_t benchres2 = BMK_benchMemInvertible(buf, ctx, BASE_CLEVEL, &cParams, BMK_compressOnly, BMK_iterMode, 1);
+        BMK_return_t benchres2 = BMK_benchMemInvertible(buf, ctx, 0, &cParams, BMK_compressOnly, BMK_iterMode, 1);
         if(benchres2.error) {
             return ERROR_RESULT;
         }
         benchres = benchres2;
     }
     if(loopDurationD < TIMELOOP_NANOSEC / 10) {
-        BMK_return_t benchres2 = BMK_benchMemInvertible(buf, ctx, BASE_CLEVEL, &cParams, BMK_decodeOnly, BMK_iterMode, 1);
+        BMK_return_t benchres2 = BMK_benchMemInvertible(buf, ctx, 0, &cParams, BMK_decodeOnly, BMK_iterMode, 1);
         if(benchres2.error) {
             return ERROR_RESULT;
         }
@@ -1420,7 +1419,7 @@ static int allBench(BMK_result_t* resultPtr,
 
     /* Final full run if estimates are unclear */
     if(loopDurationC < TIMELOOP_NANOSEC) {
-        BMK_return_t benchres2 = BMK_benchMemInvertible(buf, ctx, BASE_CLEVEL, &cParams, BMK_compressOnly, BMK_timeMode, 1);
+        BMK_return_t benchres2 = BMK_benchMemInvertible(buf, ctx, 0, &cParams, BMK_compressOnly, BMK_timeMode, 1);
         if(benchres2.error) {
             return ERROR_RESULT;
         }
@@ -1428,7 +1427,7 @@ static int allBench(BMK_result_t* resultPtr,
     } 
 
     if(loopDurationD < TIMELOOP_NANOSEC) {
-        BMK_return_t benchres2 = BMK_benchMemInvertible(buf, ctx, BASE_CLEVEL, &cParams, BMK_decodeOnly, BMK_timeMode, 1);
+        BMK_return_t benchres2 = BMK_benchMemInvertible(buf, ctx, 0, &cParams, BMK_decodeOnly, BMK_timeMode, 1);
         if(benchres2.error) {
             return ERROR_RESULT;
         }

From 0cea75402478ded21d2300544bcb538f4d3e3786 Mon Sep 17 00:00:00 2001
From: George Lu 
Date: Mon, 13 Aug 2018 16:15:34 -0700
Subject: [PATCH 143/372] Revert "Reorder declaration"

This reverts commit 3ac2c22485ab5508f47e3eab642b787af0e68b5f.
---
 tests/paramgrill.c | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/tests/paramgrill.c b/tests/paramgrill.c
index bcb6bcae3..e530af735 100644
--- a/tests/paramgrill.c
+++ b/tests/paramgrill.c
@@ -2256,9 +2256,8 @@ int main(int argc, const char** argv)
                             g_params.strategy = (ZSTD_strategy)readU32FromChar(&argument);
                             continue;
                         case 'L':
-                            {   int cLevel;
-                                argument++;
-                                cLevel = readU32FromChar(&argument);
+                            {   argument++;
+                                int const cLevel = readU32FromChar(&argument);
                                 g_params = ZSTD_getCParams(cLevel, g_blockSize, 0);
                                 continue;
                             }

From 13611249a5d1426e4902fe32b1cf913228f246fa Mon Sep 17 00:00:00 2001
From: George Lu 
Date: Tue, 24 Jul 2018 17:26:21 -0700
Subject: [PATCH 144/372] Table

Compiling
+Euclidean Metric
---
 tests/paramgrill.c | 186 +++++++++++++++++++++++++++++++++++++++++++--
 1 file changed, 179 insertions(+), 7 deletions(-)

diff --git a/tests/paramgrill.c b/tests/paramgrill.c
index e530af735..5c654d993 100644
--- a/tests/paramgrill.c
+++ b/tests/paramgrill.c
@@ -279,6 +279,29 @@ static int compareResultLT(const BMK_result_t result1, const BMK_result_t result
 
 }
 
+/* calculates normalized euclidean distance of result1 if it is in the first quadrant relative to lvlRes */
+static double resultDistLvl(const BMK_result_t result1, const BMK_result_t lvlRes) {
+    double normalizedCSpeedGain1 = result1.cSpeed / lvlRes.cSpeed - 1;
+    double normalizedRatioGain1 = lvlRes.cSize / result1.cSize - 1;
+    if(normalizedRatioGain1 < 0 || normalizedRatioGain1 < 0) {
+        return 0.0;
+    }
+    return normalizedRatioGain1 * normalizedRatioGain1 + normalizedCSpeedGain1 * normalizedCSpeedGain1;
+}
+
+static int lvlFeasible(const BMK_result_t result, const BMK_result_t lvlRes) {
+    return lvlRes.cSpeed < result.cSpeed && lvlRes.cSize > result.cSize;
+}
+
+/* redefines feasibility for lvl mode */
+static int compareResultLT2(const BMK_result_t result1, const BMK_result_t result2, const BMK_result_t lvltarget, size_t srcSize) {
+    constraint_t target = { (U32)lvltarget.cSpeed, 0, (U32)-1 };
+    if(lvlFeasible(result1, lvltarget) && lvlFeasible(result2, lvltarget)) {
+        return resultDistLvl(result1, lvltarget) < resultDistLvl(result2, lvltarget);
+    }
+    return lvlFeasible(result2, lvltarget) || (!lvlFeasible(result1, lvltarget) && (resultScore(result1, srcSize, target) < resultScore(result2, srcSize, target)));
+}
+
 /* factor sort of arbitrary */
 static constraint_t relaxTarget(constraint_t target) {
     target.cMem = (U32)-1;
@@ -629,6 +652,132 @@ static BMK_return_t BMK_benchMemInvertible(const buffers_t buf, const contexts_t
 }
 
 
+typedef struct ll_node ll_node;
+struct ll_node {
+    winnerInfo_t res;
+    ll_node* next;
+};
+
+static ll_node* g_winners; /* linked list sorted ascending by cSize & cSpeed */
+static BMK_result_t g_lvltarget;
+
+/* comparison function: */
+/* strictly better, strictly worse, equal, speed-side adv, size-side adv */
+//Maybe use compress_only for benchmark first run?
+#define WORSE_RESULT 0
+#define BETTER_RESULT 1
+#define ERROR_RESULT 2
+
+#define SPEED_RESULT 4
+#define SIZE_RESULT 5
+static int speedSizeCompare(BMK_result_t r1, BMK_result_t r2) {
+    if(r1.cSpeed > r2.cSpeed) {
+        if(r1.cSize <= r2.cSize) {
+            return WORSE_RESULT;
+        }
+        return SIZE_RESULT; /* r2 is smaller but not faster. */
+    } else {
+        if(r1.cSize >= r2.cSize) {
+            return BETTER_RESULT;
+        }
+        return SPEED_RESULT; /* r2 is faster but not smaller */
+    }
+}
+/* assumes candidate is already strictly better than old winner. */
+/* 0 for success, 1 for no insert */
+/* indicate whether inserted as well? */
+/* maintain invariant speedSizeCompare(n, n->next) = SPEED_RESULT */
+static int insertWinner(winnerInfo_t w) {
+    BMK_result_t r = w.result;
+    ll_node* cur_node = g_winners;
+    /* first node to insert */
+    if(!lvlFeasible(r, g_lvltarget)) {
+        return 1;
+    }
+
+    if(g_winners == NULL) {
+        ll_node* first_node = malloc(sizeof(ll_node));
+        if(first_node == NULL) {
+            return 1;
+        }
+        first_node->next = NULL;
+        first_node->res = w;
+        g_winners = first_node;
+        return 0;
+    }
+
+    while(cur_node->next != NULL) {
+        switch(speedSizeCompare(r, cur_node->res.result)) {
+            case BETTER_RESULT:
+            {
+                return 1; /* never insert if better */
+            }
+            case WORSE_RESULT:
+            {
+                ll_node* tmp;
+                cur_node->res = cur_node->next->res;
+                tmp = cur_node->next;
+                cur_node->next = cur_node->next->next;
+                free(tmp);
+                break; 
+            }
+            case SPEED_RESULT:
+                cur_node = cur_node->next;
+            case SIZE_RESULT: /* insert after first size result, then return */
+            {
+                ll_node* newnode = malloc(sizeof(ll_node));
+                if(newnode == NULL) {
+                    return 1;
+                }
+                newnode->res = cur_node->res;
+                cur_node->res = w;
+                newnode->next = cur_node->next;
+                cur_node->next = newnode;
+                return 0;
+            }
+        } 
+
+    }
+
+    //assert(cur_node->next == NULL)
+    switch(speedSizeCompare(r, cur_node->res.result)) {
+        case BETTER_RESULT:
+        {
+            return 1; /* never insert if better */
+        }
+        case WORSE_RESULT:
+        {
+            cur_node->res = w;
+            return 0;
+        }
+        case SPEED_RESULT:
+        {
+            ll_node* newnode = malloc(sizeof(ll_node));
+            if(newnode == NULL) {
+                return 1;
+            }
+            newnode->res = w;
+            newnode->next = NULL;
+            cur_node->next = newnode;
+            return 0;
+        }
+        case SIZE_RESULT: /* insert before first size result, then return */
+        {
+            ll_node* newnode = malloc(sizeof(ll_node));
+            if(newnode == NULL) {
+                return 1;
+            }
+            newnode->res = cur_node->res;
+            cur_node->res = w;
+            newnode->next = cur_node->next;
+            cur_node->next = newnode;
+            return 0;
+        }
+        default: 
+            return 1;
+    } 
+}
+
 static void BMK_printWinner(FILE* f, const U32 cLevel, const BMK_result_t result, const ZSTD_compressionParameters params, const size_t srcSize)
 {
     char lvlstr[15] = "Custom Level";
@@ -658,6 +807,29 @@ static void BMK_printWinnerOpt(FILE* f, const U32 cLevel, const BMK_result_t res
     /* global winner used for constraints */
     static winnerInfo_t g_winner = { { 0, 0, (size_t)-1, (size_t)-1 } , { 0, 0, 0, 0, 0, 0, ZSTD_fast } }; 
     
+    /* print lvl if optmode */
+    if(g_lvltarget.cSize != 0) {
+        winnerInfo_t w;
+        ll_node* n;
+        int i;
+        w.result = result;
+        w.params = params;
+        i = insertWinner(w);
+        if(i) return;
+
+        fprintf(f, "\033c"); 
+        for(n = g_winners; n != NULL; n = n->next) {
+             DISPLAY("\r%79s\r", "");
+             fprintf(f,"    {%3u,%3u,%3u,%3u,%3u,%3u, %s },  ",
+                params.windowLog, params.chainLog, params.hashLog, params.searchLog, params.searchLength,
+                params.targetLength, g_stratName[(U32)(params.strategy)]);
+            fprintf(f,
+            "   /* R:%5.3f at %5.1f MB/s - %5.1f MB/s */\n",
+            (double)srcSize / result.cSize, result.cSpeed / (1 << 20), result.dSpeed / (1 << 20));
+        }
+        return;
+    }
+
     if(DEBUG || compareResultLT(g_winner.result, result, targetConstraints, srcSize)) {
         if(DEBUG && compareResultLT(g_winner.result, result, targetConstraints, srcSize)) {
             DISPLAY("New Winner: \n");
@@ -1334,12 +1506,6 @@ int benchFiles(const char** fileNamesTable, int nbFiles)
     return 0;
 }
 
-
-
-#define WORSE_RESULT 0
-#define BETTER_RESULT 1
-#define ERROR_RESULT 2
-
 /* Benchmarking which stops when we are sufficiently sure the solution is infeasible / worse than the winner */
 #define VARIANCE 1.1 
 static int allBench(BMK_result_t* resultPtr,
@@ -1939,6 +2105,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_
         goto _cleanUp;
     }
  
+    /* use level'ing mode instead of normal target mode */
     if(cLevel) {
         winner.params = ZSTD_getCParams(cLevel, maxBlockSize, ctx.dictSize);
         if(BMK_benchParam(&winner.result, buf, ctx, winner.params)) {
@@ -1947,6 +2114,11 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_
         }
 
         target.cSpeed = (U32)winner.result.cSpeed;
+
+        g_targetConstraints = target;
+        
+        g_lvltarget = winner.result;
+
         BMK_printWinnerOpt(stdout, cLevel, winner.result, winner.params, target, buf.srcSize);
     }
 
@@ -2137,8 +2309,8 @@ int main(int argc, const char** argv)
     U32 main_pause = 0;
     int optimizerCLevel = 0;
 
-
     constraint_t target = { 0, 0, (U32)-1 }; 
+
     ZSTD_compressionParameters paramTarget = { 0, 0, 0, 0, 0, 0, 0 };
 
     assert(argc>=1);   /* for exename */

From 5f4502fc0775f10175e759a7987289a80d9dbd92 Mon Sep 17 00:00:00 2001
From: George Lu 
Date: Tue, 24 Jul 2018 17:55:17 -0700
Subject: [PATCH 145/372] New climb

feas part 2 uses euclidean metric
---
 tests/paramgrill.c | 31 ++++++++++++++++++++++---------
 1 file changed, 22 insertions(+), 9 deletions(-)

diff --git a/tests/paramgrill.c b/tests/paramgrill.c
index 5c654d993..ffb5c1f7d 100644
--- a/tests/paramgrill.c
+++ b/tests/paramgrill.c
@@ -1518,6 +1518,7 @@ static int allBench(BMK_result_t* resultPtr,
     U64 loopDurationC = 0, loopDurationD = 0;
     double uncertaintyConstantC, uncertaintyConstantD;
     double winnerRS;
+    int lvlmode = g_lvltarget.cSize != 0;
 
     /* initial benchmarking, gives exact ratio and memory, warms up future runs */
     benchres = BMK_benchMemInvertible(buf, ctx, 0, &cParams, BMK_both, BMK_iterMode, 1);
@@ -1548,9 +1549,11 @@ static int allBench(BMK_result_t* resultPtr,
         uncertaintyConstantD = 3;
     }
 
+    if(!lvlmode) {
     /* anything with worse ratio in feas is definitely worse, discard */
-    if(feas && benchres.result.cSize < winnerResult->cSize) {
-        return WORSE_RESULT;
+        if(feas && benchres.result.cSize < winnerResult->cSize) {
+            return WORSE_RESULT;
+        }
     }
 
     /* second run, if first run is too short, gives approximate cSpeed + dSpeed */
@@ -1578,8 +1581,9 @@ static int allBench(BMK_result_t* resultPtr,
 
     /* disregard infeasible results in feas mode */
     /* disregard if resultMax < winner in infeas mode */
-    if((feas && !feasible(resultMax, target)) || 
-      (!feas && (winnerRS > resultScore(resultMax, buf.srcSize, target)))) {
+    if((feas && (!lvlmode && !feasible(resultMax, target))) ||
+      (!feas && ((!lvlmode && winnerRS > resultScore(resultMax, buf.srcSize, target)) || 
+        (lvlmode && resultDistLvl(*winnerResult, g_lvltarget) > resultDistLvl(resultMax, g_lvltarget))))) {
         return WORSE_RESULT;
     }
 
@@ -1604,11 +1608,20 @@ static int allBench(BMK_result_t* resultPtr,
 
     /* compare by resultScore when in infeas */
     /* compare by compareResultLT when in feas */
-    if((!feas && (resultScore(benchres.result, buf.srcSize, target) > resultScore(*winnerResult, buf.srcSize, target))) || 
-       (feas && (compareResultLT(*winnerResult, benchres.result, target, buf.srcSize))) )  { 
-        return BETTER_RESULT; 
-    } else { 
-        return WORSE_RESULT; 
+    if(!lvlmode) {
+        if((!feas && (resultScore(benchres.result, buf.srcSize, target) > resultScore(*winnerResult, buf.srcSize, target))) || 
+           (feas && (compareResultLT(*winnerResult, benchres.result, target, buf.srcSize))) )  { 
+            return BETTER_RESULT; 
+        } else { 
+            return WORSE_RESULT; 
+        }
+    } else {
+        if((feas && (compareResultLT2(*winnerResult, benchres.result, g_lvltarget, buf.srcSize))) || 
+            (!feas && (resultScore(benchres.result, buf.srcSize, target) > resultScore(*winnerResult, buf.srcSize, target)))) {
+            return BETTER_RESULT;
+        } else {
+            return WORSE_RESULT;
+        }
     }
 
 }

From f67d040c39af10f68d7b6c61b06e72e1273551ff Mon Sep 17 00:00:00 2001
From: George Lu 
Date: Wed, 25 Jul 2018 11:37:20 -0700
Subject: [PATCH 146/372] Bugfixes, style changes

Complete euclidean distance climb
---
 tests/paramgrill.c | 191 +++++++++++++++++++++++++--------------------
 1 file changed, 108 insertions(+), 83 deletions(-)

diff --git a/tests/paramgrill.c b/tests/paramgrill.c
index ffb5c1f7d..0dbe66534 100644
--- a/tests/paramgrill.c
+++ b/tests/paramgrill.c
@@ -112,6 +112,36 @@ static U32 g_noSeed = 0;
 static ZSTD_compressionParameters g_params = { 0, 0, 0, 0, 0, 0, ZSTD_greedy };
 static UTIL_time_t g_time; /* to be used to compare solution finding speeds to compare to original */
 
+
+typedef struct {
+    BMK_result_t result;
+    ZSTD_compressionParameters params;
+} winnerInfo_t;
+
+/* global winner used for display. */
+//Should be totally 0 initialized? 
+static winnerInfo_t g_winner = { { 0, 0, (size_t)-1, (size_t)-1 } , { 0, 0, 0, 0, 0, 0, ZSTD_fast } }; 
+
+typedef struct {
+    U32 cSpeed; /* bytes / sec */
+    U32 dSpeed;
+    U32 cMem;    /* bytes */    
+} constraint_t;
+
+static constraint_t g_targetConstraints; 
+
+typedef struct ll_node ll_node;
+struct ll_node {
+    winnerInfo_t res;
+    ll_node* next;
+};
+
+static ll_node* g_winners; /* linked list sorted ascending by cSize & cSpeed */
+static BMK_result_t g_lvltarget;
+
+/* range 0 - 99 */
+static U32 g_strictness = 99;
+
 void BMK_SetNbIterations(int nbLoops)
 {
     g_nbIterations = nbLoops;
@@ -197,12 +227,6 @@ static void findClockGranularity(void) {
     DEBUGOUTPUT("Granularity: %llu\n", (unsigned long long)g_clockGranularity);
 }
 
-typedef struct {
-    U32 cSpeed; /* bytes / sec */
-    U32 dSpeed;
-    U32 cMem;    /* bytes */    
-} constraint_t;
-
 #define CLAMPCHECK(val,min,max) {                     \
     if (val && (((val)<(min)) | ((val)>(max)))) {     \
         DISPLAY("INVALID PARAMETER CONSTRAINTS\n");   \
@@ -246,7 +270,7 @@ static void BMK_translateAdvancedParams(const ZSTD_compressionParameters params)
 
 /* checks results are feasible */
 static int feasible(const BMK_result_t results, const constraint_t target) {
-    return (results.cSpeed >= target.cSpeed) && (results.dSpeed >= target.dSpeed) && (results.cMem <= target.cMem);
+    return (results.cSpeed >= target.cSpeed) && (results.dSpeed >= target.dSpeed) && (results.cMem <= target.cMem) && (!g_lvltarget.cSize || results.cSize <= g_lvltarget.cSize);
 }
 
 /* hill climbing value for part 1 */
@@ -269,44 +293,33 @@ static double resultScore(const BMK_result_t res, const size_t srcSize, const co
     return ret;
 }
 
-/* return true if r2 strictly better than r1 */ 
-static int compareResultLT(const BMK_result_t result1, const BMK_result_t result2, const constraint_t target, size_t srcSize) {
-    if(feasible(result1, target) && feasible(result2, target)) {
-        return (result1.cSize > result2.cSize) || (result1.cSize == result2.cSize && result2.cSpeed > result1.cSpeed)
-        || (result1.cSize == result2.cSize && result2.cSpeed == result1.cSpeed && result2.dSpeed > result1.dSpeed);
-    }
-    return feasible(result2, target) || (!feasible(result1, target) && (resultScore(result1, srcSize, target) < resultScore(result2, srcSize, target)));
-
-}
-
 /* calculates normalized euclidean distance of result1 if it is in the first quadrant relative to lvlRes */
 static double resultDistLvl(const BMK_result_t result1, const BMK_result_t lvlRes) {
-    double normalizedCSpeedGain1 = result1.cSpeed / lvlRes.cSpeed - 1;
-    double normalizedRatioGain1 = lvlRes.cSize / result1.cSize - 1;
-    if(normalizedRatioGain1 < 0 || normalizedRatioGain1 < 0) {
+    double normalizedCSpeedGain1 = (result1.cSpeed / lvlRes.cSpeed) - 1;
+    double normalizedRatioGain1 = ((double)lvlRes.cSize / result1.cSize) - 1;
+    if(normalizedRatioGain1 < 0 || normalizedCSpeedGain1 < 0) {
         return 0.0;
     }
     return normalizedRatioGain1 * normalizedRatioGain1 + normalizedCSpeedGain1 * normalizedCSpeedGain1;
 }
 
-static int lvlFeasible(const BMK_result_t result, const BMK_result_t lvlRes) {
-    return lvlRes.cSpeed < result.cSpeed && lvlRes.cSize > result.cSize;
-}
-
-/* redefines feasibility for lvl mode */
-static int compareResultLT2(const BMK_result_t result1, const BMK_result_t result2, const BMK_result_t lvltarget, size_t srcSize) {
-    constraint_t target = { (U32)lvltarget.cSpeed, 0, (U32)-1 };
-    if(lvlFeasible(result1, lvltarget) && lvlFeasible(result2, lvltarget)) {
-        return resultDistLvl(result1, lvltarget) < resultDistLvl(result2, lvltarget);
+/* return true if r2 strictly better than r1 */ 
+static int compareResultLT(const BMK_result_t result1, const BMK_result_t result2, const constraint_t target, size_t srcSize) {
+    if(feasible(result1, target) && feasible(result2, target)) {
+        if(g_lvltarget.cSize == 0) {
+            return (result1.cSize > result2.cSize) || (result1.cSize == result2.cSize && result2.cSpeed > result1.cSpeed)
+            || (result1.cSize == result2.cSize && result2.cSpeed == result1.cSpeed && result2.dSpeed > result1.dSpeed);
+        } else {
+            return resultDistLvl(result1, g_lvltarget) < resultDistLvl(result2, g_lvltarget);
+        }
     }
-    return lvlFeasible(result2, lvltarget) || (!lvlFeasible(result1, lvltarget) && (resultScore(result1, srcSize, target) < resultScore(result2, srcSize, target)));
+    return feasible(result2, target) || (!feasible(result1, target) && (resultScore(result1, srcSize, target) < resultScore(result2, srcSize, target)));
 }
 
-/* factor sort of arbitrary */
 static constraint_t relaxTarget(constraint_t target) {
     target.cMem = (U32)-1;
-    target.cSpeed *= 0.9; 
-    target.dSpeed *= 0.9;
+    target.cSpeed *= ((double)(g_strictness + 1) / 100); 
+    target.dSpeed *= ((double)(g_strictness + 1) / 100);
     return target;
 }
 
@@ -330,11 +343,6 @@ BMK_benchParam1(BMK_result_t* resultPtr,
     return res.error;
 }
 
-typedef struct {
-    BMK_result_t result;
-    ZSTD_compressionParameters params;
-} winnerInfo_t;
-
 static ZSTD_compressionParameters emptyParams(void) {
     ZSTD_compressionParameters p = { 0, 0, 0, 0, 0, 0, (ZSTD_strategy)0 };
     return p;
@@ -651,16 +659,6 @@ static BMK_return_t BMK_benchMemInvertible(const buffers_t buf, const contexts_t
     return results;
 }
 
-
-typedef struct ll_node ll_node;
-struct ll_node {
-    winnerInfo_t res;
-    ll_node* next;
-};
-
-static ll_node* g_winners; /* linked list sorted ascending by cSize & cSpeed */
-static BMK_result_t g_lvltarget;
-
 /* comparison function: */
 /* strictly better, strictly worse, equal, speed-side adv, size-side adv */
 //Maybe use compress_only for benchmark first run?
@@ -691,7 +689,7 @@ static int insertWinner(winnerInfo_t w) {
     BMK_result_t r = w.result;
     ll_node* cur_node = g_winners;
     /* first node to insert */
-    if(!lvlFeasible(r, g_lvltarget)) {
+    if(!feasible(r, g_targetConstraints)) {
         return 1;
     }
 
@@ -722,7 +720,10 @@ static int insertWinner(winnerInfo_t w) {
                 break; 
             }
             case SPEED_RESULT:
+            {
                 cur_node = cur_node->next;
+                break;
+            }
             case SIZE_RESULT: /* insert after first size result, then return */
             {
                 ll_node* newnode = malloc(sizeof(ll_node));
@@ -843,6 +844,34 @@ static void BMK_printWinnerOpt(FILE* f, const U32 cLevel, const BMK_result_t res
             g_winner.params = params;
         }
     }  
+
+    //prints out tradeoff table if using lvl
+    if(g_lvltarget.cSize != 0) {
+        winnerInfo_t w;
+        ll_node* n;
+        int i;
+        w.result = result;
+        w.params = params;
+        i = insertWinner(w);
+        if(i) return;
+
+        if(!DEBUG) { fprintf(f, "\033c"); }
+        fprintf(f, "\n"); 
+
+        /* the table */
+        fprintf(f, "================================\n");
+        for(n = g_winners; n != NULL; n = n->next) {
+            DISPLAY("\r%79s\r", "");
+
+            fprintf(f,"    {%3u,%3u,%3u,%3u,%3u,%3u, %s },  ",
+                n->res.params.windowLog, n->res.params.chainLog, n->res.params.hashLog, n->res.params.searchLog, n->res.params.searchLength,
+                n->res.params.targetLength, g_stratName[(U32)(n->res.params.strategy)]);
+            fprintf(f,
+            "   /* R:%5.3f at %5.1f MB/s - %5.1f MB/s */\n",
+            (double)srcSize / n->res.result.cSize, n->res.result.cSpeed / (1 << 20), n->res.result.dSpeed / (1 << 20));
+        }
+        fprintf(f, "================================\n");
+    }
 }
 
 static void BMK_printWinners2(FILE* f, const winnerInfo_t* winners, size_t srcSize)
@@ -1518,7 +1547,6 @@ static int allBench(BMK_result_t* resultPtr,
     U64 loopDurationC = 0, loopDurationD = 0;
     double uncertaintyConstantC, uncertaintyConstantD;
     double winnerRS;
-    int lvlmode = g_lvltarget.cSize != 0;
 
     /* initial benchmarking, gives exact ratio and memory, warms up future runs */
     benchres = BMK_benchMemInvertible(buf, ctx, 0, &cParams, BMK_both, BMK_iterMode, 1);
@@ -1549,11 +1577,9 @@ static int allBench(BMK_result_t* resultPtr,
         uncertaintyConstantD = 3;
     }
 
-    if(!lvlmode) {
     /* anything with worse ratio in feas is definitely worse, discard */
-        if(feas && benchres.result.cSize < winnerResult->cSize) {
-            return WORSE_RESULT;
-        }
+    if(feas && benchres.result.cSize < winnerResult->cSize && g_lvltarget.cSize == 0) {
+        return WORSE_RESULT;
     }
 
     /* second run, if first run is too short, gives approximate cSpeed + dSpeed */
@@ -1581,9 +1607,8 @@ static int allBench(BMK_result_t* resultPtr,
 
     /* disregard infeasible results in feas mode */
     /* disregard if resultMax < winner in infeas mode */
-    if((feas && (!lvlmode && !feasible(resultMax, target))) ||
-      (!feas && ((!lvlmode && winnerRS > resultScore(resultMax, buf.srcSize, target)) || 
-        (lvlmode && resultDistLvl(*winnerResult, g_lvltarget) > resultDistLvl(resultMax, g_lvltarget))))) {
+    if((feas && !feasible(resultMax, target)) ||
+      (!feas && (winnerRS > resultScore(resultMax, buf.srcSize, target)))) {
         return WORSE_RESULT;
     }
 
@@ -1608,22 +1633,12 @@ static int allBench(BMK_result_t* resultPtr,
 
     /* compare by resultScore when in infeas */
     /* compare by compareResultLT when in feas */
-    if(!lvlmode) {
-        if((!feas && (resultScore(benchres.result, buf.srcSize, target) > resultScore(*winnerResult, buf.srcSize, target))) || 
-           (feas && (compareResultLT(*winnerResult, benchres.result, target, buf.srcSize))) )  { 
-            return BETTER_RESULT; 
-        } else { 
-            return WORSE_RESULT; 
-        }
-    } else {
-        if((feas && (compareResultLT2(*winnerResult, benchres.result, g_lvltarget, buf.srcSize))) || 
-            (!feas && (resultScore(benchres.result, buf.srcSize, target) > resultScore(*winnerResult, buf.srcSize, target)))) {
-            return BETTER_RESULT;
-        } else {
-            return WORSE_RESULT;
-        }
+    if((!feas && (resultScore(benchres.result, buf.srcSize, target) > resultScore(*winnerResult, buf.srcSize, target))) || 
+       (feas && (compareResultLT(*winnerResult, benchres.result, target, buf.srcSize))) )  { 
+        return BETTER_RESULT; 
+    } else { 
+        return WORSE_RESULT; 
     }
-
 }
 
 #define INFEASIBLE_THRESHOLD 200
@@ -1655,6 +1670,7 @@ static int benchMemo(BMK_result_t* resultPtr,
     return res;
 }
 
+
 /* One iteration of hill climbing. Specifically, it first tries all 
  * valid parameter configurations w/ manhattan distance 1 and picks the best one
  * failing that, it progressively tries candidates further and further away (up to #dim + 2)
@@ -1667,6 +1683,10 @@ static int benchMemo(BMK_result_t* resultPtr,
  * Phase 2 optimizes in accordance with what the original function sets out to maximize, with
  * all feasible solutions valued over all infeasible solutions.
  */
+
+/* sanitize all params here. 
+ * all generation after random should be sanitized. (maybe sanitize random)
+ */
 static winnerInfo_t climbOnce(const constraint_t target, 
                 const varInds_t* varArray, const int varLen, 
                 U8* const memoTable,
@@ -1800,6 +1820,7 @@ static winnerInfo_t optimizeFixedStrategy(
             BMK_printWinnerOpt(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, target, buf.srcSize);
             i = 0;
         }
+
         i++;
     }
     return winnerInfo;
@@ -2129,8 +2150,10 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_
         target.cSpeed = (U32)winner.result.cSpeed;
 
         g_targetConstraints = target;
-        
-        g_lvltarget = winner.result;
+
+        g_lvltarget = winner.result; 
+        g_lvltarget.cSpeed *= ((double)(g_strictness + 1) / 100);
+        g_lvltarget.cSize /= ((double)(g_strictness + 1) / 100);
 
         BMK_printWinnerOpt(stdout, cLevel, winner.result, winner.params, target, buf.srcSize);
     }
@@ -2299,15 +2322,16 @@ static int badusage(const char* exename)
 }
 
 #define PARSE_SUB_ARGS(stringLong, stringShort, variable) { if (longCommandWArg(&argument, stringLong) || longCommandWArg(&argument, stringShort)) { variable = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; } }
-#define PARSE_CPARAMS(variable)                                          \
-{                                                                        \
-    PARSE_SUB_ARGS("windowLog=",     "wlog=",  variable.windowLog);      \
-    PARSE_SUB_ARGS("chainLog=" ,     "clog=",  variable.chainLog);       \
-    PARSE_SUB_ARGS("hashLog=",       "hlog=",  variable.hashLog);        \
-    PARSE_SUB_ARGS("searchLog=" ,    "slog=",  variable.searchLog);      \
-    PARSE_SUB_ARGS("searchLength=",  "slen=",  variable.searchLength);   \
-    PARSE_SUB_ARGS("targetLength=" , "tlen=",  variable.targetLength);   \
-    PARSE_SUB_ARGS("strategy=",      "strat=", variable.strategy);       \
+#define PARSE_CPARAMS(variable)                                            \
+{                                                                          \
+    PARSE_SUB_ARGS("windowLog=",       "wlog=",  variable.vals[wlog_ind]); \
+    PARSE_SUB_ARGS("chainLog=" ,       "clog=",  variable.vals[clog_ind]); \
+    PARSE_SUB_ARGS("hashLog=",         "hlog=",  variable.vals[hlog_ind]); \
+    PARSE_SUB_ARGS("searchLog=" ,      "slog=",  variable.vals[slog_ind]); \
+    PARSE_SUB_ARGS("searchLength=",    "slen=",  variable.vals[slen_ind]); \
+    PARSE_SUB_ARGS("targetLength=" ,   "tlen=",  variable.vals[tlen_ind]); \
+    PARSE_SUB_ARGS("strategy=",        "strat=", variable.vals[strt_ind]); \
+    PARSE_SUB_ARGS("forceAttachDict=", "fad="  , variable.vals[strt_ind]); \
 }
 
 int main(int argc, const char** argv)
@@ -2350,6 +2374,7 @@ int main(int argc, const char** argv)
                 PARSE_SUB_ARGS("decompressionSpeed=", "dSpeed=", target.dSpeed); 
                 PARSE_SUB_ARGS("compressionMemory=" , "cMem=", target.cMem);
                 PARSE_SUB_ARGS("level=", "lvl=", optimizerCLevel);
+                PARSE_SUB_ARGS("strict=", "stc=", g_strictness);
                 DISPLAY("invalid optimization parameter \n");
                 return 1;
             }

From 2bdfe6ca71d2171c02468214759080bc1d3e7e3a Mon Sep 17 00:00:00 2001
From: George Lu 
Date: Wed, 25 Jul 2018 11:55:09 -0700
Subject: [PATCH 147/372] Better Display

---
 tests/paramgrill.c | 34 ++++++++++++++++++++++++++++------
 1 file changed, 28 insertions(+), 6 deletions(-)

diff --git a/tests/paramgrill.c b/tests/paramgrill.c
index 0dbe66534..fa5a5390a 100644
--- a/tests/paramgrill.c
+++ b/tests/paramgrill.c
@@ -293,7 +293,7 @@ static double resultScore(const BMK_result_t res, const size_t srcSize, const co
     return ret;
 }
 
-/* calculates normalized euclidean distance of result1 if it is in the first quadrant relative to lvlRes */
+/* calculates normalized squared euclidean distance of result1 if it is in the first quadrant relative to lvlRes */
 static double resultDistLvl(const BMK_result_t result1, const BMK_result_t lvlRes) {
     double normalizedCSpeedGain1 = (result1.cSpeed / lvlRes.cSpeed) - 1;
     double normalizedRatioGain1 = ((double)lvlRes.cSize / result1.cSize) - 1;
@@ -668,6 +668,7 @@ static BMK_return_t BMK_benchMemInvertible(const buffers_t buf, const contexts_t
 
 #define SPEED_RESULT 4
 #define SIZE_RESULT 5
+/* maybe have epsilon-eq to limit table size? */
 static int speedSizeCompare(BMK_result_t r1, BMK_result_t r2) {
     if(r1.cSpeed > r2.cSpeed) {
         if(r1.cSize <= r2.cSize) {
@@ -853,11 +854,11 @@ static void BMK_printWinnerOpt(FILE* f, const U32 cLevel, const BMK_result_t res
         w.result = result;
         w.params = params;
         i = insertWinner(w);
-        if(i) return;
+        //if(i) return;
 
         if(!DEBUG) { fprintf(f, "\033c"); }
         fprintf(f, "\n"); 
-
+        
         /* the table */
         fprintf(f, "================================\n");
         for(n = g_winners; n != NULL; n = n->next) {
@@ -871,6 +872,27 @@ static void BMK_printWinnerOpt(FILE* f, const U32 cLevel, const BMK_result_t res
             (double)srcSize / n->res.result.cSize, n->res.result.cSpeed / (1 << 20), n->res.result.dSpeed / (1 << 20));
         }
         fprintf(f, "================================\n");
+        fprintf(f, "Level Bounds: R: > %.3f AND C: < %.1f MB/s \n\n",
+            (double)srcSize / g_lvltarget.cSize, g_lvltarget.cSpeed / (1 << 20));
+
+
+        fprintf(f, "Overall Winner: \n");
+        fprintf(f,"    {%3u,%3u,%3u,%3u,%3u,%3u, %s },  ",
+            g_winner.params.windowLog, g_winner.params.chainLog, g_winner.params.hashLog, g_winner.params.searchLog, g_winner.params.searchLength,
+            g_winner.params.targetLength, g_stratName[(U32)(g_winner.params.strategy)]);
+        fprintf(f,
+        "   /* R:%5.3f at %5.1f MB/s - %5.1f MB/s */\n",
+        (double)srcSize / g_winner.result.cSize, g_winner.result.cSpeed / (1 << 20), g_winner.result.dSpeed / (1 << 20));
+
+
+        fprintf(f, "Latest BMK: \n");
+        fprintf(f,"    {%3u,%3u,%3u,%3u,%3u,%3u, %s },  ",
+            params.windowLog, params.chainLog, params.hashLog, params.searchLog, params.searchLength,
+            params.targetLength, g_stratName[(U32)(params.strategy)]);
+        fprintf(f,
+            "   /* R:%5.3f at %5.1f MB/s - %5.1f MB/s */\n",
+            (double)srcSize / result.cSize, result.cSpeed / (1 << 20), result.dSpeed / (1 << 20));
+
     }
 }
 
@@ -1707,7 +1729,7 @@ static winnerInfo_t climbOnce(const constraint_t target,
 
     {
         winnerInfo_t bestFeasible1 = initWinnerInfo(cparam);
-        DISPLAY("Climb Part 1\n");
+        DEBUGOUTPUT("Climb Part 1\n");
         while(better) {
 
             int i, dist, offset;
@@ -1774,7 +1796,7 @@ static winnerInfo_t climbOnce(const constraint_t target,
                 feas = 1;
                 better = 1;
                 winnerInfo = bestFeasible1; /* note with change, bestFeasible may not necessarily be feasible, but if one has been benchmarked, it will be. */
-                DISPLAY("Climb Part 2\n");
+                DEBUGOUTPUT("Climb Part 2\n");
             }
         }
         winnerInfo = bestFeasible1;
@@ -2218,9 +2240,9 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_
                 }
 
                 while(st && tries) {
+                    DEBUGOUTPUT("StrategySwitch: %s\n", g_stratName[st]);
                     winnerInfo_t wc = optimizeFixedStrategy(buf, ctx, target, paramTarget, 
                         st, varArray, varLen, allMT[st], tries);
-                    DEBUGOUTPUT("StratNum %d\n", st);
                     if(compareResultLT(winner.result, wc.result, target, buf.srcSize)) {
                         winner = wc;
                     }

From 3a2e95eba45a28afa57afae00dac15dce29959e9 Mon Sep 17 00:00:00 2001
From: George Lu 
Date: Thu, 26 Jul 2018 16:45:00 -0700
Subject: [PATCH 148/372] Perf improvements

try decay
strategy selection skipping
---
 tests/paramgrill.c | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/tests/paramgrill.c b/tests/paramgrill.c
index fa5a5390a..0db8510e1 100644
--- a/tests/paramgrill.c
+++ b/tests/paramgrill.c
@@ -2079,7 +2079,9 @@ static ZSTD_compressionParameters maskParams(ZSTD_compressionParameters base, ZS
     return base;
 }
 
+/* experiment with playing with this and decay value */
 #define MAX_TRIES 8
+#define TRY_DECAY 3
 /* main fn called when using --optimize */
 /* Does strategy selection by benchmarking default compression levels
  * then optimizes by strategy, starting with the best one and moving 
@@ -2092,6 +2094,7 @@ static ZSTD_compressionParameters maskParams(ZSTD_compressionParameters base, ZS
  * paramTarget - parameter constraints (i.e. restriction search space to where strategy = ZSTD_fast)
  * cLevel - compression level to exceed (all solutions must be > lvl in cSpeed + ratio)
  */
+
 static int optimizeForSize(const char* const * const fileNamesTable, const size_t nbFiles, const char* dictFileName, constraint_t target, ZSTD_compressionParameters paramTarget, int cLevel)
 {
     varInds_t varArray [NUM_PARAMS];
@@ -2202,7 +2205,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_
             /* strategy selection */
             const int maxSeeds = g_noSeed ? 1 : ZSTD_maxCLevel();
             DEBUGOUTPUT("Strategy Selection\n");
-            if(paramTarget.strategy == 0) { /* no variable based constraints */  
+            if(paramTarget.strategy == 0) {
                 BMK_result_t candidate;
                 int i;
                 for (i=1; i<=maxSeeds; i++) {
@@ -2216,6 +2219,11 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_
                         winner.result = candidate;
                         winner.params = CParams;
                     }
+
+                    /* if the current params are too slow, just stop. */
+                    if(target.cSpeed != 0 && target.cSpeed > winner.result.cSpeed / 2) {
+                        break;
+                    }
                 }
             }
         }
@@ -2248,7 +2256,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_
                     }
 
                     st = nextStrategy(st, bestStrategy);
-                    tries--;
+                    tries -= TRY_DECAY;
                 }
             } else {
                 winner = optimizeFixedStrategy(buf, ctx, target, paramTarget, paramTarget.strategy, 

From 8ff0de15e4484b656dc199e118b063b3665cb2dd Mon Sep 17 00:00:00 2001
From: George Lu 
Date: Fri, 27 Jul 2018 08:20:31 -0700
Subject: [PATCH 149/372] Generalize, macro magic numbers

---
 tests/README.md    |  3 +++
 tests/paramgrill.c | 54 +++++++++++++++++++++++++++++++---------------
 2 files changed, 40 insertions(+), 17 deletions(-)

diff --git a/tests/README.md b/tests/README.md
index 2f0026fda..04eb5094e 100644
--- a/tests/README.md
+++ b/tests/README.md
@@ -112,6 +112,9 @@ Full list of arguments
     dSpeed= - Minimum decompression speed
     cMem= - compression memory
     lvl= - Automatically sets compression speed constraint to the speed of that level
+    stc= - In lvl mode, represents slack in ratio/cSpeed allowed for a solution to be considered
+         - In normal operation, represents slack in strategy selection in choosing the default parameters
+ --optimize=  : same as -O with more verbose syntax 
  -P#          : generated sample compressibility 
  -t#          : Caps runtime of operation in seconds (default : 99999 seconds (about 27 hours )) 
  -v           : Prints Benchmarking output
diff --git a/tests/paramgrill.c b/tests/paramgrill.c
index 0db8510e1..fca6b2002 100644
--- a/tests/paramgrill.c
+++ b/tests/paramgrill.c
@@ -139,8 +139,9 @@ struct ll_node {
 static ll_node* g_winners; /* linked list sorted ascending by cSize & cSpeed */
 static BMK_result_t g_lvltarget;
 
-/* range 0 - 99 */
-static U32 g_strictness = 99;
+/* range 0 - 99, measure of how strict  */
+#define DEFAULT_STRICTNESS 99999
+static U32 g_strictness = DEFAULT_STRICTNESS;
 
 void BMK_SetNbIterations(int nbLoops)
 {
@@ -318,8 +319,8 @@ static int compareResultLT(const BMK_result_t result1, const BMK_result_t result
 
 static constraint_t relaxTarget(constraint_t target) {
     target.cMem = (U32)-1;
-    target.cSpeed *= ((double)(g_strictness + 1) / 100); 
-    target.dSpeed *= ((double)(g_strictness + 1) / 100);
+    target.cSpeed *= ((double)(g_strictness) / 100); 
+    target.dSpeed *= ((double)(g_strictness) / 100);
     return target;
 }
 
@@ -2080,8 +2081,7 @@ static ZSTD_compressionParameters maskParams(ZSTD_compressionParameters base, ZS
 }
 
 /* experiment with playing with this and decay value */
-#define MAX_TRIES 8
-#define TRY_DECAY 3
+
 /* main fn called when using --optimize */
 /* Does strategy selection by benchmarking default compression levels
  * then optimizes by strategy, starting with the best one and moving 
@@ -2095,6 +2095,9 @@ static ZSTD_compressionParameters maskParams(ZSTD_compressionParameters base, ZS
  * cLevel - compression level to exceed (all solutions must be > lvl in cSpeed + ratio)
  */
 
+#define MAX_TRIES 3
+#define TRY_DECAY 1
+
 static int optimizeForSize(const char* const * const fileNamesTable, const size_t nbFiles, const char* dictFileName, constraint_t target, ZSTD_compressionParameters paramTarget, int cLevel)
 {
     varInds_t varArray [NUM_PARAMS];
@@ -2164,6 +2167,21 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_
         goto _cleanUp;
     }
  
+    /* default strictness = Maximum for */
+    if(g_strictness == DEFAULT_STRICTNESS) {
+        if(cLevel) {
+            g_strictness = 99;
+        } else {
+            g_strictness = 90;
+        }
+    } else {
+        if(0 >= g_strictness || g_strictness > 100) {
+            DISPLAY("Strictness Outside of Bounds\n");
+            ret = 4;
+            goto _cleanUp;
+        }
+    }
+
     /* use level'ing mode instead of normal target mode */
     if(cLevel) {
         winner.params = ZSTD_getCParams(cLevel, maxBlockSize, ctx.dictSize);
@@ -2177,8 +2195,8 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_
         g_targetConstraints = target;
 
         g_lvltarget = winner.result; 
-        g_lvltarget.cSpeed *= ((double)(g_strictness + 1) / 100);
-        g_lvltarget.cSize /= ((double)(g_strictness + 1) / 100);
+        g_lvltarget.cSpeed *= ((double)(g_strictness) / 100);
+        g_lvltarget.cSize /= ((double)(g_strictness) / 100);
 
         BMK_printWinnerOpt(stdout, cLevel, winner.result, winner.params, target, buf.srcSize);
     }
@@ -2199,6 +2217,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_
 
     {   
         varInds_t varNew[NUM_PARAMS];
+        ZSTD_compressionParameters CParams;
 
         /* find best solution from default params */
         {
@@ -2210,8 +2229,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_
                 int i;
                 for (i=1; i<=maxSeeds; i++) {
                     int ec;
-                    ZSTD_compressionParameters CParams = ZSTD_getCParams(i, maxBlockSize, ctx.dictSize);
-                    CParams = maskParams(CParams, paramTarget);
+                    CParams = maskParams(ZSTD_getCParams(i, maxBlockSize, ctx.dictSize), paramTarget);
                     ec = BMK_benchParam(&candidate, buf, ctx, CParams);
                     BMK_printWinnerOpt(stdout, i, candidate, CParams, target, buf.srcSize);
 
@@ -2221,9 +2239,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_
                     }
 
                     /* if the current params are too slow, just stop. */
-                    if(target.cSpeed != 0 && target.cSpeed > winner.result.cSpeed / 2) {
-                        break;
-                    }
+                    if(target.cSpeed > candidate.cSpeed * 2) { break; }
                 }
             }
         }
@@ -2239,6 +2255,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_
                 int tries = MAX_TRIES;
 
                 { 
+                    /* one iterations of hill climbing with the level-defined parameters. */
                     int varLenNew = sanitizeVarArray(varNew, varLen, varArray, st);
                     winnerInfo_t w1 = climbOnce(target, varNew, varLenNew, allMT[st], 
                         buf, ctx, winner.params);
@@ -2247,16 +2264,19 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_
                     }
                 }
 
-                while(st && tries) {
+                while(st && tries > 0) {
                     DEBUGOUTPUT("StrategySwitch: %s\n", g_stratName[st]);
                     winnerInfo_t wc = optimizeFixedStrategy(buf, ctx, target, paramTarget, 
                         st, varArray, varLen, allMT[st], tries);
+
                     if(compareResultLT(winner.result, wc.result, target, buf.srcSize)) {
                         winner = wc;
+                        tries = MAX_TRIES;
+                        bestStrategy = st;
+                    } else {
+                        st = nextStrategy(st, bestStrategy);
+                        tries -= TRY_DECAY;
                     }
-
-                    st = nextStrategy(st, bestStrategy);
-                    tries -= TRY_DECAY;
                 }
             } else {
                 winner = optimizeFixedStrategy(buf, ctx, target, paramTarget, paramTarget.strategy, 

From b3544217b77bcff66b7bf754a4f7b745a7d0493a Mon Sep 17 00:00:00 2001
From: George Lu 
Date: Fri, 27 Jul 2018 11:47:14 -0700
Subject: [PATCH 150/372] Cleanup

---
 tests/README.md    |  6 ++++--
 tests/paramgrill.c | 30 ++++++++++++++++++++----------
 2 files changed, 24 insertions(+), 12 deletions(-)

diff --git a/tests/README.md b/tests/README.md
index 04eb5094e..1410ca979 100644
--- a/tests/README.md
+++ b/tests/README.md
@@ -111,9 +111,11 @@ Full list of arguments
     cSpeed= - Minimum compression speed
     dSpeed= - Minimum decompression speed
     cMem= - compression memory
-    lvl= - Automatically sets compression speed constraint to the speed of that level
-    stc= - In lvl mode, represents slack in ratio/cSpeed allowed for a solution to be considered
+    lvl= - Searches for solutions which are strictly better than that compression lvl in ratio and cSpeed, 
+    stc= - When invoked with lvl=, represents slack in ratio/cSpeed allowed for a solution to be considered
          - In normal operation, represents slack in strategy selection in choosing the default parameters
+    prefer[Speed/Ratio]= - Only affects lvl= invocations. Defines value placed on compression speed or ratio
+          when determining overall winner (default 1 for both).
  --optimize=  : same as -O with more verbose syntax 
  -P#          : generated sample compressibility 
  -t#          : Caps runtime of operation in seconds (default : 99999 seconds (about 27 hours )) 
diff --git a/tests/paramgrill.c b/tests/paramgrill.c
index fca6b2002..188b04935 100644
--- a/tests/paramgrill.c
+++ b/tests/paramgrill.c
@@ -123,7 +123,7 @@ typedef struct {
 static winnerInfo_t g_winner = { { 0, 0, (size_t)-1, (size_t)-1 } , { 0, 0, 0, 0, 0, 0, ZSTD_fast } }; 
 
 typedef struct {
-    U32 cSpeed; /* bytes / sec */
+    U32 cSpeed;  /* bytes / sec */
     U32 dSpeed;
     U32 cMem;    /* bytes */    
 } constraint_t;
@@ -138,6 +138,12 @@ struct ll_node {
 
 static ll_node* g_winners; /* linked list sorted ascending by cSize & cSpeed */
 static BMK_result_t g_lvltarget;
+static int g_optmode = 0;
+
+static U32 g_speedMultiplier = 1;
+static U32 g_ratioMultiplier = 1;
+
+/* g_mode? */
 
 /* range 0 - 99, measure of how strict  */
 #define DEFAULT_STRICTNESS 99999
@@ -271,7 +277,7 @@ static void BMK_translateAdvancedParams(const ZSTD_compressionParameters params)
 
 /* checks results are feasible */
 static int feasible(const BMK_result_t results, const constraint_t target) {
-    return (results.cSpeed >= target.cSpeed) && (results.dSpeed >= target.dSpeed) && (results.cMem <= target.cMem) && (!g_lvltarget.cSize || results.cSize <= g_lvltarget.cSize);
+    return (results.cSpeed >= target.cSpeed) && (results.dSpeed >= target.dSpeed) && (results.cMem <= target.cMem) && (!g_optmode || results.cSize <= g_lvltarget.cSize);
 }
 
 /* hill climbing value for part 1 */
@@ -291,6 +297,7 @@ static double resultScore(const BMK_result_t res, const size_t srcSize, const co
 
     ret = (MIN(1, cs) + MIN(1, ds)  + MIN(1, cm))*r1 + rt * rtr + 
          (MAX(0, log(cs))+ MAX(0, log(ds))+ MAX(0, log(cm))) * r2;
+
     return ret;
 }
 
@@ -301,17 +308,17 @@ static double resultDistLvl(const BMK_result_t result1, const BMK_result_t lvlRe
     if(normalizedRatioGain1 < 0 || normalizedCSpeedGain1 < 0) {
         return 0.0;
     }
-    return normalizedRatioGain1 * normalizedRatioGain1 + normalizedCSpeedGain1 * normalizedCSpeedGain1;
+    return normalizedRatioGain1 * g_ratioMultiplier + normalizedCSpeedGain1 * g_speedMultiplier;
 }
 
 /* return true if r2 strictly better than r1 */ 
 static int compareResultLT(const BMK_result_t result1, const BMK_result_t result2, const constraint_t target, size_t srcSize) {
     if(feasible(result1, target) && feasible(result2, target)) {
-        if(g_lvltarget.cSize == 0) {
+        if(g_optmode) {
+            return resultDistLvl(result1, g_lvltarget) < resultDistLvl(result2, g_lvltarget);
+        } else {
             return (result1.cSize > result2.cSize) || (result1.cSize == result2.cSize && result2.cSpeed > result1.cSpeed)
             || (result1.cSize == result2.cSize && result2.cSpeed == result1.cSpeed && result2.dSpeed > result1.dSpeed);
-        } else {
-            return resultDistLvl(result1, g_lvltarget) < resultDistLvl(result2, g_lvltarget);
         }
     }
     return feasible(result2, target) || (!feasible(result1, target) && (resultScore(result1, srcSize, target) < resultScore(result2, srcSize, target)));
@@ -848,7 +855,7 @@ static void BMK_printWinnerOpt(FILE* f, const U32 cLevel, const BMK_result_t res
     }  
 
     //prints out tradeoff table if using lvl
-    if(g_lvltarget.cSize != 0) {
+    if(g_optmode) {
         winnerInfo_t w;
         ll_node* n;
         int i;
@@ -1601,7 +1608,7 @@ static int allBench(BMK_result_t* resultPtr,
     }
 
     /* anything with worse ratio in feas is definitely worse, discard */
-    if(feas && benchres.result.cSize < winnerResult->cSize && g_lvltarget.cSize == 0) {
+    if(feas && benchres.result.cSize < winnerResult->cSize && !g_optmode) {
         return WORSE_RESULT;
     }
 
@@ -2169,7 +2176,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_
  
     /* default strictness = Maximum for */
     if(g_strictness == DEFAULT_STRICTNESS) {
-        if(cLevel) {
+        if(g_optmode) {
             g_strictness = 99;
         } else {
             g_strictness = 90;
@@ -2183,7 +2190,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_
     }
 
     /* use level'ing mode instead of normal target mode */
-    if(cLevel) {
+    if(g_optmode) {
         winner.params = ZSTD_getCParams(cLevel, maxBlockSize, ctx.dictSize);
         if(BMK_benchParam(&winner.result, buf, ctx, winner.params)) {
             ret = 3;
@@ -2425,6 +2432,9 @@ int main(int argc, const char** argv)
                 PARSE_SUB_ARGS("compressionMemory=" , "cMem=", target.cMem);
                 PARSE_SUB_ARGS("level=", "lvl=", optimizerCLevel);
                 PARSE_SUB_ARGS("strict=", "stc=", g_strictness);
+                PARSE_SUB_ARGS("preferSpeed=", "prfSpd=", g_speedMultiplier);
+                PARSE_SUB_ARGS("preferRatio=", "prfRto=", g_ratioMultiplier);
+
                 DISPLAY("invalid optimization parameter \n");
                 return 1;
             }

From a884b76bc2fedd086c36b16e52b0dc43d075ca28 Mon Sep 17 00:00:00 2001
From: George Lu 
Date: Fri, 27 Jul 2018 14:19:55 -0700
Subject: [PATCH 151/372] Style Changes

Add single run dictionaries
Change MB to be consistent 1 << 20 rather than 1,000,000
---
 tests/paramgrill.c | 481 ++++++++++++++++++++++-----------------------
 1 file changed, 238 insertions(+), 243 deletions(-)

diff --git a/tests/paramgrill.c b/tests/paramgrill.c
index 188b04935..8df72eb01 100644
--- a/tests/paramgrill.c
+++ b/tests/paramgrill.c
@@ -107,6 +107,7 @@ static double g_compressibility = COMPRESSIBILITY_DEFAULT;
 static U32 g_blockSize = 0;
 static U32 g_rand = 1;
 static U32 g_singleRun = 0;
+static U32 g_optimizer = 0;
 static U32 g_target = 0;
 static U32 g_noSeed = 0;
 static ZSTD_compressionParameters g_params = { 0, 0, 0, 0, 0, 0, ZSTD_greedy };
@@ -118,10 +119,6 @@ typedef struct {
     ZSTD_compressionParameters params;
 } winnerInfo_t;
 
-/* global winner used for display. */
-//Should be totally 0 initialized? 
-static winnerInfo_t g_winner = { { 0, 0, (size_t)-1, (size_t)-1 } , { 0, 0, 0, 0, 0, 0, ZSTD_fast } }; 
-
 typedef struct {
     U32 cSpeed;  /* bytes / sec */
     U32 dSpeed;
@@ -386,8 +383,7 @@ typedef struct {
     ZSTD_DCtx* dctx;
 } contexts_t;
 
-static int
-BMK_benchParam(BMK_result_t* resultPtr,
+static int BMK_benchParam(BMK_result_t* resultPtr,
                 const buffers_t buf, const contexts_t ctx,
                 const ZSTD_compressionParameters cParams) {
     BMK_return_t res = BMK_benchMem(buf.srcPtrs[0], buf.srcSize, buf.srcSizes, (unsigned)buf.nbBlocks, 0, &cParams, ctx.dictBuffer, ctx.dictSize, 0, "Files");
@@ -520,6 +516,175 @@ static size_t local_defaultDecompress(
 *  From Paramgrill End
 *********************************************************/
 
+static void freeBuffers(const buffers_t b) {
+    if(b.srcPtrs != NULL) {
+        free(b.srcBuffer);
+    }
+    free(b.srcPtrs);
+    free(b.srcSizes);
+
+    if(b.dstPtrs != NULL) {
+        free(b.dstPtrs[0]);
+    }
+    free(b.dstPtrs);
+    free(b.dstCapacities);
+    free(b.dstSizes);
+
+    if(b.resPtrs != NULL) {
+        free(b.resPtrs[0]);
+    }
+    free(b.resPtrs);
+}
+
+/* allocates buffer's arguments. returns success / failuere */
+static int createBuffers(buffers_t* buff, const char* const * const fileNamesTable, 
+                          const size_t nbFiles)
+{
+    size_t pos = 0;
+    size_t n;
+    U64 const totalSizeToLoad = UTIL_getTotalFileSize(fileNamesTable, (U32)nbFiles);
+    size_t benchedSize = MIN(BMK_findMaxMem(totalSizeToLoad * 3) / 3, totalSizeToLoad);
+    const size_t blockSize = g_blockSize ? g_blockSize : totalSizeToLoad; //(largest fileSize or total fileSize)
+    U32 const maxNbBlocks = (U32) ((totalSizeToLoad + (blockSize-1)) / blockSize) + (U32)nbFiles;
+    U32 blockNb = 0;
+
+    buff->srcPtrs = (const void**)calloc(maxNbBlocks, sizeof(void*));
+    buff->srcSizes = (size_t*)malloc(maxNbBlocks * sizeof(size_t));
+
+    buff->dstPtrs = (void**)calloc(maxNbBlocks, sizeof(void*));
+    buff->dstCapacities = (size_t*)malloc(maxNbBlocks * sizeof(size_t));
+    buff->dstSizes = (size_t*)malloc(maxNbBlocks * sizeof(size_t));
+
+    buff->resPtrs = (void**)calloc(maxNbBlocks, sizeof(void*));
+    buff->resSizes = (size_t*)malloc(maxNbBlocks * sizeof(size_t)); 
+
+    if(!buff->srcPtrs || !buff->srcSizes || !buff->dstPtrs || !buff->dstCapacities || !buff->dstSizes || !buff->resPtrs || !buff->resSizes) {
+        DISPLAY("alloc error\n");
+        freeBuffers(*buff);
+        return 1;
+    }
+
+    buff->srcBuffer = malloc(benchedSize);
+    buff->srcPtrs[0] = (const void*)buff->srcBuffer;
+    buff->dstPtrs[0] = malloc(ZSTD_compressBound(benchedSize) + (maxNbBlocks * 1024));
+    buff->resPtrs[0] = malloc(benchedSize);
+
+    if(!buff->srcPtrs[0] || !buff->dstPtrs[0] || !buff->resPtrs[0]) {
+        DISPLAY("alloc error\n");
+        freeBuffers(*buff);
+        return 1;
+    }
+
+    for(n = 0; n < nbFiles; n++) {
+        FILE* f;
+        U64 fileSize = UTIL_getFileSize(fileNamesTable[n]);
+        if (UTIL_isDirectory(fileNamesTable[n])) {
+            DISPLAY("Ignoring %s directory...       \n", fileNamesTable[n]);
+            continue;
+        }
+        if (fileSize == UTIL_FILESIZE_UNKNOWN) {
+            DISPLAY("Cannot evaluate size of %s, ignoring ... \n", fileNamesTable[n]);
+            continue;
+        }
+        f = fopen(fileNamesTable[n], "rb");
+        if (f==NULL) {
+            DISPLAY("impossible to open file %s\n", fileNamesTable[n]);
+            freeBuffers(*buff);
+            fclose(f);
+            return 10;
+        }
+
+        DISPLAY("Loading %s...       \r", fileNamesTable[n]);
+
+        if (fileSize + pos > benchedSize) fileSize = benchedSize - pos, n=nbFiles;   /* buffer too small - stop after this file */
+        {
+            char* buffer = (char*)(buff->srcBuffer); 
+            size_t const readSize = fread((buffer)+pos, 1, (size_t)fileSize, f);
+            size_t blocked = 0;
+            while(blocked < readSize) {
+                buff->srcPtrs[blockNb] = (const void*)((buffer) + (pos + blocked));
+                buff->srcSizes[blockNb] = blockSize;
+                blocked += blockSize;
+                blockNb++;
+            }
+            if(readSize > 0) { buff->srcSizes[blockNb - 1] = ((readSize - 1) % blockSize) + 1; }
+
+            if (readSize != (size_t)fileSize) {
+                DISPLAY("could not read %s", fileNamesTable[n]);
+                freeBuffers(*buff);
+                fclose(f);
+                return 1;
+            }
+
+            pos += readSize;
+
+        }
+        fclose(f);
+    }
+
+    buff->dstCapacities[0] = ZSTD_compressBound(buff->srcSizes[0]);
+    buff->dstSizes[0] = buff->dstCapacities[0];
+    buff->resSizes[0] = buff->srcSizes[0];
+
+    for(n = 1; n < blockNb; n++) {
+        buff->dstPtrs[n] = ((char*)buff->dstPtrs[n-1]) + buff->dstCapacities[n-1];
+        buff->resPtrs[n] = ((char*)buff->resPtrs[n-1]) + buff->resSizes[n-1];
+        buff->dstCapacities[n] = ZSTD_compressBound(buff->srcSizes[n]);
+        buff->dstSizes[n] = buff->dstCapacities[n];
+        buff->resSizes[n] = buff->srcSizes[n];
+    }
+    buff->srcSize = pos;
+    buff->nbBlocks = blockNb;
+
+    if (pos == 0) { DISPLAY("\nno data to bench\n"); return 1; }
+
+    return 0;
+}
+
+static void freeContexts(const contexts_t ctx) {
+    free(ctx.dictBuffer);
+    ZSTD_freeCCtx(ctx.cctx);
+    ZSTD_freeDCtx(ctx.dctx);
+}
+
+static int createContexts(contexts_t* ctx, const char* dictFileName) {
+    FILE* f;
+    size_t readSize;
+    ctx->cctx = ZSTD_createCCtx();
+    ctx->dctx = ZSTD_createDCtx();
+    if(dictFileName == NULL) {
+        ctx->dictSize = 0;
+        ctx->dictBuffer = NULL;
+        return 0;
+    }
+    ctx->dictSize = UTIL_getFileSize(dictFileName);
+    ctx->dictBuffer = malloc(ctx->dictSize);
+
+    f = fopen(dictFileName, "rb");
+    
+    if(!f) {
+        DISPLAY("unable to open file\n");
+        fclose(f);
+        freeContexts(*ctx);
+        return 1;
+    }
+    
+    if(ctx->dictSize > 64 MB || !(ctx->dictBuffer)) {
+        DISPLAY("dictionary too large\n");
+        fclose(f);
+        freeContexts(*ctx);
+        return 1;
+    }
+    readSize = fread(ctx->dictBuffer, 1, ctx->dictSize, f);
+    if(readSize != ctx->dictSize) {
+        DISPLAY("unable to read file\n");
+        fclose(f);
+        freeContexts(*ctx);
+        return 1;
+    }
+    return 0;
+}
+
 /* Replicate functionality of benchMemAdvanced, but with pre-split src / dst buffers */
 /* The purpose is so that sufficient information is returned so that a decompression call to benchMemInvertible is possible */
 /* BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, fileSizes, nbFiles, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); */
@@ -788,6 +953,8 @@ static int insertWinner(winnerInfo_t w) {
     } 
 }
 
+/* Writes to f the results of a parameter benchmark */
+/* when used with --optimize, will only print results better than previously discovered */
 static void BMK_printWinner(FILE* f, const U32 cLevel, const BMK_result_t result, const ZSTD_compressionParameters params, const size_t srcSize)
 {
     char lvlstr[15] = "Custom Level";
@@ -806,7 +973,7 @@ static void BMK_printWinner(FILE* f, const U32 cLevel, const BMK_result_t result
 
     fprintf(f,
         "/* %s */   /* R:%5.3f at %5.1f MB/s - %5.1f MB/s */",
-        lvlstr, (double)srcSize / result.cSize, (double)result.cSpeed / (1 << 20), (double)result.dSpeed / (1 << 20));
+        lvlstr, (double)srcSize / result.cSize, (double)result.cSpeed / (1 MB), (double)result.dSpeed / (1 MB));
 
     if(TIMED) { fprintf(f, " - %1lu:%2lu:%05.2f", (unsigned long) minutes / 60,(unsigned long) minutes % 60, (double)(time - minutes * TIMELOOP_NANOSEC * 60ULL)/TIMELOOP_NANOSEC); }
     fprintf(f, "\n"); 
@@ -816,29 +983,6 @@ static void BMK_printWinnerOpt(FILE* f, const U32 cLevel, const BMK_result_t res
 {
     /* global winner used for constraints */
     static winnerInfo_t g_winner = { { 0, 0, (size_t)-1, (size_t)-1 } , { 0, 0, 0, 0, 0, 0, ZSTD_fast } }; 
-    
-    /* print lvl if optmode */
-    if(g_lvltarget.cSize != 0) {
-        winnerInfo_t w;
-        ll_node* n;
-        int i;
-        w.result = result;
-        w.params = params;
-        i = insertWinner(w);
-        if(i) return;
-
-        fprintf(f, "\033c"); 
-        for(n = g_winners; n != NULL; n = n->next) {
-             DISPLAY("\r%79s\r", "");
-             fprintf(f,"    {%3u,%3u,%3u,%3u,%3u,%3u, %s },  ",
-                params.windowLog, params.chainLog, params.hashLog, params.searchLog, params.searchLength,
-                params.targetLength, g_stratName[(U32)(params.strategy)]);
-            fprintf(f,
-            "   /* R:%5.3f at %5.1f MB/s - %5.1f MB/s */\n",
-            (double)srcSize / result.cSize, result.cSpeed / (1 << 20), result.dSpeed / (1 << 20));
-        }
-        return;
-    }
 
     if(DEBUG || compareResultLT(g_winner.result, result, targetConstraints, srcSize)) {
         if(DEBUG && compareResultLT(g_winner.result, result, targetConstraints, srcSize)) {
@@ -855,14 +999,12 @@ static void BMK_printWinnerOpt(FILE* f, const U32 cLevel, const BMK_result_t res
     }  
 
     //prints out tradeoff table if using lvl
-    if(g_optmode) {
+    if(g_optmode && g_optimizer) {
         winnerInfo_t w;
         ll_node* n;
-        int i;
         w.result = result;
         w.params = params;
-        i = insertWinner(w);
-        //if(i) return;
+        insertWinner(w);
 
         if(!DEBUG) { fprintf(f, "\033c"); }
         fprintf(f, "\n"); 
@@ -877,11 +1019,11 @@ static void BMK_printWinnerOpt(FILE* f, const U32 cLevel, const BMK_result_t res
                 n->res.params.targetLength, g_stratName[(U32)(n->res.params.strategy)]);
             fprintf(f,
             "   /* R:%5.3f at %5.1f MB/s - %5.1f MB/s */\n",
-            (double)srcSize / n->res.result.cSize, n->res.result.cSpeed / (1 << 20), n->res.result.dSpeed / (1 << 20));
+            (double)srcSize / n->res.result.cSize, (double)n->res.result.cSpeed / (1 MB), (double)n->res.result.dSpeed / (1 MB));
         }
         fprintf(f, "================================\n");
         fprintf(f, "Level Bounds: R: > %.3f AND C: < %.1f MB/s \n\n",
-            (double)srcSize / g_lvltarget.cSize, g_lvltarget.cSpeed / (1 << 20));
+            (double)srcSize / g_lvltarget.cSize, (double)g_lvltarget.cSpeed / (1 MB));
 
 
         fprintf(f, "Overall Winner: \n");
@@ -890,8 +1032,9 @@ static void BMK_printWinnerOpt(FILE* f, const U32 cLevel, const BMK_result_t res
             g_winner.params.targetLength, g_stratName[(U32)(g_winner.params.strategy)]);
         fprintf(f,
         "   /* R:%5.3f at %5.1f MB/s - %5.1f MB/s */\n",
-        (double)srcSize / g_winner.result.cSize, g_winner.result.cSpeed / (1 << 20), g_winner.result.dSpeed / (1 << 20));
+        (double)srcSize / g_winner.result.cSize, (double)g_winner.result.cSpeed / (1 MB), (double)g_winner.result.dSpeed / (1 MB));
 
+        BMK_translateAdvancedParams(g_winner.params);
 
         fprintf(f, "Latest BMK: \n");
         fprintf(f,"    {%3u,%3u,%3u,%3u,%3u,%3u, %s },  ",
@@ -899,7 +1042,7 @@ static void BMK_printWinnerOpt(FILE* f, const U32 cLevel, const BMK_result_t res
             params.targetLength, g_stratName[(U32)(params.strategy)]);
         fprintf(f,
             "   /* R:%5.3f at %5.1f MB/s - %5.1f MB/s */\n",
-            (double)srcSize / result.cSize, result.cSpeed / (1 << 20), result.dSpeed / (1 << 20));
+            (double)srcSize / result.cSize, (double)result.cSpeed / (1 MB), (double)result.dSpeed / (1 MB));
 
     }
 }
@@ -1023,16 +1166,16 @@ static int BMK_seed(winnerInfo_t* winners, const ZSTD_compressionParameters para
                 /* too large compression speed difference for the compression benefit */
                 if (W_ratio > O_ratio)
                 DISPLAY ("Compression Speed : %5.3f @ %4.1f MB/s  vs  %5.3f @ %4.1f MB/s   : not enough for level %i\n",
-                         W_ratio, (double)testResult.cSpeed / 1000000,
-                         O_ratio, (double)winners[cLevel].result.cSpeed / 1000000.,   cLevel);
+                         W_ratio, (double)testResult.cSpeed / (1 MB),
+                         O_ratio, (double)winners[cLevel].result.cSpeed / (1 MB),   cLevel);
                 continue;
             }
             if (W_DSpeed_note   < O_DSpeed_note  ) {
                 /* too large decompression speed difference for the compression benefit */
                 if (W_ratio > O_ratio)
                 DISPLAY ("Decompression Speed : %5.3f @ %4.1f MB/s  vs  %5.3f @ %4.1f MB/s   : not enough for level %i\n",
-                         W_ratio, (double)testResult.dSpeed / 1000000.,
-                         O_ratio, (double)winners[cLevel].result.dSpeed / 1000000.,   cLevel);
+                         W_ratio, (double)testResult.dSpeed / (1 MB),
+                         O_ratio, (double)winners[cLevel].result.dSpeed / (1 MB),   cLevel);
                 continue;
             }
 
@@ -1417,7 +1560,6 @@ static void BMK_selectRandomStart(
     }
 }
 
-
 static void BMK_benchOnce(const void* srcBuffer, size_t srcSize)
 {
     BMK_result_t testResult;
@@ -1442,7 +1584,7 @@ static void BMK_benchFullTable(const void* srcBuffer, size_t srcSize)
     if (f==NULL) { DISPLAY("error opening %s \n", rfName); exit(1); }
 
     if (g_target) {
-        BMK_init_level_constraints(g_target*1000000);
+        BMK_init_level_constraints(g_target * (1 MB));
     } else {
         /* baseline config for level 1 */
         ZSTD_compressionParameters const l1params = ZSTD_getCParams(1, blockSize, 0);
@@ -1487,7 +1629,7 @@ static void BMK_benchMemInit(const void* srcBuffer, size_t srcSize)
 static int benchSample(void)
 {
     const char* const name = "Sample 10MB";
-    size_t const benchedSize = 10000000;
+    size_t const benchedSize = 10 MB;
 
     void* origBuff = malloc(benchedSize);
     if (!origBuff) { perror("not enough memory"); return 12; }
@@ -1505,13 +1647,56 @@ static int benchSample(void)
 }
 
 
+static int benchOnce(const char** fileNamesTable, int nbFiles, const char* dictFileName) {
+    buffers_t buf;
+    contexts_t ctx;
+    BMK_result_t testResult;
+    size_t maxBlockSize = 0, i;
+
+    if(createBuffers(&buf, fileNamesTable, nbFiles)) {
+        DISPLAY("unable to load files\n");
+        return 1;
+    }
+
+    if(createContexts(&ctx, dictFileName)) {
+        DISPLAY("unable to load dictionary\n");
+        freeBuffers(buf);
+        return 2;
+    }
+
+    for(i = 0; i < buf.nbBlocks; i++) {
+        maxBlockSize = MAX(maxBlockSize, buf.srcSizes[i]);
+    }
+
+    g_params = ZSTD_adjustCParams(g_params, maxBlockSize, 0);
+
+    if(BMK_benchParam(&testResult, buf, ctx, g_params)) {
+        DISPLAY("Error during benchmarking\n");
+        freeBuffers(buf);
+        freeContexts(ctx);
+        return 3;
+    }
+
+    DISPLAY("Compression Ratio: %.3f  Compress Speed: %.1f MB/s Decompress Speed: %.1f MB/s\n", (double)buf.srcSize / testResult.cSize, 
+        (double)testResult.cSpeed / (1 MB), (double)testResult.dSpeed / (1 MB));
+
+    freeBuffers(buf);
+    freeContexts(ctx);
+    return 0;
+}
+
 /* benchFiles() :
  * note: while this function takes a table of filenames,
  * in practice, only the first filename will be used */
-int benchFiles(const char** fileNamesTable, int nbFiles)
+//TODO: dictionaries still not supported in fullTable mode
+int benchFiles(const char** fileNamesTable, int nbFiles, const char* dictFileName)
 {
     int fileIdx=0;
 
+    if(g_singleRun) {
+        return benchOnce(fileNamesTable, nbFiles, dictFileName);
+    }
+
     /* Loop for each file */
     while (fileIdxsrcPtrs = (const void**)calloc(maxNbBlocks, sizeof(void*));
-    buff->srcSizes = (size_t*)malloc(maxNbBlocks * sizeof(size_t));
-
-    buff->dstPtrs = (void**)calloc(maxNbBlocks, sizeof(void*));
-    buff->dstCapacities = (size_t*)malloc(maxNbBlocks * sizeof(size_t));
-    buff->dstSizes = (size_t*)malloc(maxNbBlocks * sizeof(size_t));
-
-    buff->resPtrs = (void**)calloc(maxNbBlocks, sizeof(void*));
-    buff->resSizes = (size_t*)malloc(maxNbBlocks * sizeof(size_t)); 
-
-    if(!buff->srcPtrs || !buff->srcSizes || !buff->dstPtrs || !buff->dstCapacities || !buff->dstSizes || !buff->resPtrs || !buff->resSizes) {
-        DISPLAY("alloc error\n");
-        freeBuffers(*buff);
-        return 1;
-    }
-
-    buff->srcBuffer = malloc(benchedSize);
-    buff->srcPtrs[0] = (const void*)buff->srcBuffer;
-    buff->dstPtrs[0] = malloc(ZSTD_compressBound(benchedSize) + (maxNbBlocks * 1024));
-    buff->resPtrs[0] = malloc(benchedSize);
-
-    if(!buff->srcPtrs[0] || !buff->dstPtrs[0] || !buff->resPtrs[0]) {
-        DISPLAY("alloc error\n");
-        freeBuffers(*buff);
-        return 1;
-    }
-
-    for(n = 0; n < nbFiles; n++) {
-        FILE* f;
-        U64 fileSize = UTIL_getFileSize(fileNamesTable[n]);
-        if (UTIL_isDirectory(fileNamesTable[n])) {
-            DISPLAY("Ignoring %s directory...       \n", fileNamesTable[n]);
-            continue;
-        }
-        if (fileSize == UTIL_FILESIZE_UNKNOWN) {
-            DISPLAY("Cannot evaluate size of %s, ignoring ... \n", fileNamesTable[n]);
-            continue;
-        }
-        f = fopen(fileNamesTable[n], "rb");
-        if (f==NULL) {
-            DISPLAY("impossible to open file %s\n", fileNamesTable[n]);
-            freeBuffers(*buff);
-            fclose(f);
-            return 10;
-        }
-
-        DISPLAY("Loading %s...       \r", fileNamesTable[n]);
-
-        if (fileSize + pos > benchedSize) fileSize = benchedSize - pos, n = nbFiles;   /* buffer too small - stop after this file */
-        {
-            char* buffer = (char*)(buff->srcBuffer); 
-            size_t const readSize = fread(((buffer)+pos), 1, (size_t)fileSize, f);
-            size_t blocked = 0;
-            while(blocked < readSize) {
-                buff->srcPtrs[blockNb] = (const void*)((buffer) + (pos + blocked));
-                buff->srcSizes[blockNb] = blockSize;
-                blocked += blockSize;
-                blockNb++;
-            }
-            if(readSize > 0) { buff->srcSizes[blockNb - 1] = ((readSize - 1) % blockSize) + 1; }
-
-            if (readSize != (size_t)fileSize) {
-                DISPLAY("could not read %s", fileNamesTable[n]);
-                freeBuffers(*buff);
-                fclose(f);
-                return 1;
-            }
-
-            pos += readSize;
-
-        }
-        fclose(f);
-    }
-
-    buff->dstCapacities[0] = ZSTD_compressBound(buff->srcSizes[0]);
-    buff->dstSizes[0] = buff->dstCapacities[0];
-    buff->resSizes[0] = buff->srcSizes[0];
-
-    for(n = 1; n < blockNb; n++) {
-        buff->dstPtrs[n] = ((char*)buff->dstPtrs[n-1]) + buff->dstCapacities[n-1];
-        buff->resPtrs[n] = ((char*)buff->resPtrs[n-1]) + buff->resSizes[n-1];
-        buff->dstCapacities[n] = ZSTD_compressBound(buff->srcSizes[n]);
-        buff->dstSizes[n] = buff->dstCapacities[n];
-        buff->resSizes[n] = buff->srcSizes[n];
-    }
-    buff->srcSize = pos;
-    buff->nbBlocks = blockNb;
-
-    if (pos == 0) { DISPLAY("\nno data to bench\n"); return 1; }
-
-    return 0;
-}
-
-static void freeContexts(const contexts_t ctx) {
-    free(ctx.dictBuffer);
-    ZSTD_freeCCtx(ctx.cctx);
-    ZSTD_freeDCtx(ctx.dctx);
-}
-
-/* Creates struct holding contexts and dictionary buffers. returns 0 on success, 1 on failure. */
-static int createContexts(contexts_t* const ctx, const char* dictFileName) {
-    FILE* f;
-    size_t readSize;
-    U64 dictSize;
-    ctx->cctx = ZSTD_createCCtx();
-    ctx->dctx = ZSTD_createDCtx();
-    ctx->dictSize = 0;
-    ctx->dictBuffer = NULL;
-
-    if(!ctx->cctx || !ctx->dctx) {
-        DISPLAY("context allocation error\n");
-        freeContexts(*ctx);
-        return 1;
-    }
-
-    if(dictFileName == NULL) {
-        return 0;
-    }
-
-    dictSize = UTIL_getFileSize(dictFileName);
-
-    if(dictSize == UTIL_FILESIZE_UNKNOWN) {
-        DISPLAY("Unable to get dictionary size\n");
-        freeContexts(*ctx);
-        return 1;
-    } else {
-        ctx->dictSize = (size_t)dictSize;
-    }
-
-    ctx->dictBuffer = malloc(ctx->dictSize);
-
-    f = fopen(dictFileName, "rb");
-    
-    if(!f) {
-        DISPLAY("unable to open file\n");
-        fclose(f);
-        freeContexts(*ctx);
-        return 1;
-    }
-    
-    if(ctx->dictSize > 64 MB || !(ctx->dictBuffer)) {
-        DISPLAY("dictionary too large\n");
-        fclose(f);
-        freeContexts(*ctx);
-        return 1;
-    }
-    readSize = fread(ctx->dictBuffer, 1, ctx->dictSize, f);
-    if(readSize != ctx->dictSize) {
-        DISPLAY("unable to read file\n");
-        fclose(f);
-        freeContexts(*ctx);
-        return 1;
-    }
-    return 0;
-}
-
 /* goes best, best-1, best+1, best-2, ... */
 /* return 0 if nothing remaining */
 static int nextStrategy(const int currentStrategy, const int bestStrategy) {
@@ -2102,7 +2097,7 @@ static ZSTD_compressionParameters maskParams(ZSTD_compressionParameters base, ZS
  * cLevel - compression level to exceed (all solutions must be > lvl in cSpeed + ratio)
  */
 
-#define MAX_TRIES 3
+#define MAX_TRIES 5
 #define TRY_DECAY 1
 
 static int optimizeForSize(const char* const * const fileNamesTable, const size_t nbFiles, const char* dictFileName, constraint_t target, ZSTD_compressionParameters paramTarget, int cLevel)
@@ -2272,8 +2267,9 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_
                 }
 
                 while(st && tries > 0) {
+                    winnerInfo_t wc;
                     DEBUGOUTPUT("StrategySwitch: %s\n", g_stratName[st]);
-                    winnerInfo_t wc = optimizeFixedStrategy(buf, ctx, target, paramTarget, 
+                    wc = optimizeFixedStrategy(buf, ctx, target, paramTarget, 
                         st, varArray, varLen, allMT[st], tries);
 
                     if(compareResultLT(winner.result, wc.result, target, buf.srcSize)) {
@@ -2399,7 +2395,6 @@ int main(int argc, const char** argv)
     const char* exename=argv[0];
     const char* input_filename = NULL;
     const char* dictFileName = NULL;
-    U32 optimizer = 0;
     U32 main_pause = 0;
     int optimizerCLevel = 0;
 
@@ -2424,7 +2419,7 @@ int main(int argc, const char** argv)
         if(!strcmp(argument,"--no-seed")) { g_noSeed = 1; continue; }
 
         if (longCommandWArg(&argument, "--optimize=")) {
-            optimizer = 1;
+            g_optimizer = 1;
             for ( ; ;) {
                 PARSE_CPARAMS(paramTarget);
                 PARSE_SUB_ARGS("compressionSpeed=" ,  "cSpeed=", target.cSpeed);
@@ -2580,17 +2575,17 @@ int main(int argc, const char** argv)
         if (!input_filename) { input_filename=argument; filenamesStart=i; continue; }
     }
     if (filenamesStart==0) {
-        if (optimizer) {
+        if (g_optimizer) {
             DISPLAY("Optimizer Expects File\n");
             return 1;
         } else {
             result = benchSample();
         }
     } else {
-        if (optimizer) {
+        if (g_optimizer) {
             result = optimizeForSize(argv+filenamesStart, argc-filenamesStart, dictFileName, target, paramTarget, optimizerCLevel);
         } else {
-            result = benchFiles(argv+filenamesStart, argc-filenamesStart);
+            result = benchFiles(argv+filenamesStart, argc-filenamesStart, dictFileName);
     }   }
 
     if (main_pause) { int unused; printf("press enter...\n"); unused = getchar(); (void)unused; }

From 43b4971ca8910ccedebb6c23a6daa89069df6a17 Mon Sep 17 00:00:00 2001
From: George Lu 
Date: Fri, 27 Jul 2018 16:49:33 -0700
Subject: [PATCH 152/372] Renames, Documentation Updates

---
 tests/README.md    | 22 +++++++-----
 tests/paramgrill.c | 84 +++++++++++++++++++++++++---------------------
 2 files changed, 58 insertions(+), 48 deletions(-)

diff --git a/tests/README.md b/tests/README.md
index 1410ca979..946f890b1 100644
--- a/tests/README.md
+++ b/tests/README.md
@@ -107,15 +107,19 @@ Full list of arguments
     L# - level
  --zstd=      : Single run, parameter selection syntax same as zstdcli
  --optimize=  : find parameters to maximize compression ratio given parameters
-    Can use all --zstd= commands to constrain the type of solution found in addition to the following constraints
-    cSpeed= - Minimum compression speed
-    dSpeed= - Minimum decompression speed
-    cMem= - compression memory
-    lvl= - Searches for solutions which are strictly better than that compression lvl in ratio and cSpeed, 
-    stc= - When invoked with lvl=, represents slack in ratio/cSpeed allowed for a solution to be considered
-         - In normal operation, represents slack in strategy selection in choosing the default parameters
-    prefer[Speed/Ratio]= - Only affects lvl= invocations. Defines value placed on compression speed or ratio
-          when determining overall winner (default 1 for both).
+                Can use all --zstd= commands to constrain the type of solution found in addition to the following constraints
+    cSpeed=   : Minimum compression speed
+    dSpeed=   : Minimum decompression speed
+    cMem=     : Maximum compression memory
+    lvl=      : Searches for solutions which are strictly better than that compression lvl in ratio and cSpeed, 
+    stc=      : When invoked with lvl=, represents percentage slack in ratio/cSpeed allowed for a solution to be considered (Default 99%)
+              : In normal operation, represents percentage slack in choosing viable starting strategy selection in choosing the default parameters
+                (Lower value will begin with stronger strategies) (Default 90%)
+    preferSpeed= / preferRatio=
+              : Only affects lvl = invocations. Defines value placed on compression speed or ratio
+                when determining overall winner (default 1 for both, higher = more valued).
+    tries=    : Maximum number of random restarts on a single strategy before switching (Default 5)
+                Higher values will make optimizer run longer, more chances to find better solution.
  --optimize=  : same as -O with more verbose syntax 
  -P#          : generated sample compressibility 
  -t#          : Caps runtime of operation in seconds (default : 99999 seconds (about 27 hours )) 
diff --git a/tests/paramgrill.c b/tests/paramgrill.c
index 8df72eb01..7e86c2e9e 100644
--- a/tests/paramgrill.c
+++ b/tests/paramgrill.c
@@ -125,15 +125,13 @@ typedef struct {
     U32 cMem;    /* bytes */    
 } constraint_t;
 
-static constraint_t g_targetConstraints; 
-
-typedef struct ll_node ll_node;
-struct ll_node {
+typedef struct winner_ll_node winner_ll_node;
+struct winner_ll_node {
     winnerInfo_t res;
-    ll_node* next;
+    winner_ll_node* next;
 };
 
-static ll_node* g_winners; /* linked list sorted ascending by cSize & cSpeed */
+static winner_ll_node* g_winners; /* linked list sorted ascending by cSize & cSpeed */
 static BMK_result_t g_lvltarget;
 static int g_optmode = 0;
 
@@ -323,8 +321,8 @@ static int compareResultLT(const BMK_result_t result1, const BMK_result_t result
 
 static constraint_t relaxTarget(constraint_t target) {
     target.cMem = (U32)-1;
-    target.cSpeed *= ((double)(g_strictness) / 100); 
-    target.dSpeed *= ((double)(g_strictness) / 100);
+    target.cSpeed *= ((double)g_strictness) / 100; 
+    target.dSpeed *= ((double)g_strictness) / 100;
     return target;
 }
 
@@ -855,20 +853,19 @@ static int speedSizeCompare(BMK_result_t r1, BMK_result_t r2) {
         return SPEED_RESULT; /* r2 is faster but not smaller */
     }
 }
-/* assumes candidate is already strictly better than old winner. */
-/* 0 for success, 1 for no insert */
-/* indicate whether inserted as well? */
+
+/* 0 for insertion, 1 for no insert */
 /* maintain invariant speedSizeCompare(n, n->next) = SPEED_RESULT */
-static int insertWinner(winnerInfo_t w) {
+static int insertWinner(winnerInfo_t w, constraint_t targetConstraints) {
     BMK_result_t r = w.result;
-    ll_node* cur_node = g_winners;
+    winner_ll_node* cur_node = g_winners;
     /* first node to insert */
-    if(!feasible(r, g_targetConstraints)) {
+    if(!feasible(r, targetConstraints)) {
         return 1;
     }
 
     if(g_winners == NULL) {
-        ll_node* first_node = malloc(sizeof(ll_node));
+        winner_ll_node* first_node = malloc(sizeof(winner_ll_node));
         if(first_node == NULL) {
             return 1;
         }
@@ -886,7 +883,7 @@ static int insertWinner(winnerInfo_t w) {
             }
             case WORSE_RESULT:
             {
-                ll_node* tmp;
+                winner_ll_node* tmp;
                 cur_node->res = cur_node->next->res;
                 tmp = cur_node->next;
                 cur_node->next = cur_node->next->next;
@@ -900,7 +897,7 @@ static int insertWinner(winnerInfo_t w) {
             }
             case SIZE_RESULT: /* insert after first size result, then return */
             {
-                ll_node* newnode = malloc(sizeof(ll_node));
+                winner_ll_node* newnode = malloc(sizeof(winner_ll_node));
                 if(newnode == NULL) {
                     return 1;
                 }
@@ -927,7 +924,7 @@ static int insertWinner(winnerInfo_t w) {
         }
         case SPEED_RESULT:
         {
-            ll_node* newnode = malloc(sizeof(ll_node));
+            winner_ll_node* newnode = malloc(sizeof(winner_ll_node));
             if(newnode == NULL) {
                 return 1;
             }
@@ -938,7 +935,7 @@ static int insertWinner(winnerInfo_t w) {
         }
         case SIZE_RESULT: /* insert before first size result, then return */
         {
-            ll_node* newnode = malloc(sizeof(ll_node));
+            winner_ll_node* newnode = malloc(sizeof(winner_ll_node));
             if(newnode == NULL) {
                 return 1;
             }
@@ -961,7 +958,7 @@ static void BMK_printWinner(FILE* f, const U32 cLevel, const BMK_result_t result
     const U64 time = UTIL_clockSpanNano(g_time);
     const U64 minutes = time / (60ULL * TIMELOOP_NANOSEC);
 
-    DISPLAY("\r%79s\r", "");
+    fprintf(f, "\r%79s\r", "");
 
     fprintf(f,"    {%3u,%3u,%3u,%3u,%3u,%3u, %s },  ",
         params.windowLog, params.chainLog, params.hashLog, params.searchLog, params.searchLength,
@@ -1001,10 +998,10 @@ static void BMK_printWinnerOpt(FILE* f, const U32 cLevel, const BMK_result_t res
     //prints out tradeoff table if using lvl
     if(g_optmode && g_optimizer) {
         winnerInfo_t w;
-        ll_node* n;
+        winner_ll_node* n;
         w.result = result;
         w.params = params;
-        insertWinner(w);
+        insertWinner(w, targetConstraints);
 
         if(!DEBUG) { fprintf(f, "\033c"); }
         fprintf(f, "\n"); 
@@ -1012,7 +1009,7 @@ static void BMK_printWinnerOpt(FILE* f, const U32 cLevel, const BMK_result_t res
         /* the table */
         fprintf(f, "================================\n");
         for(n = g_winners; n != NULL; n = n->next) {
-            DISPLAY("\r%79s\r", "");
+            fprintf(f, "\r%79s\r", "");
 
             fprintf(f,"    {%3u,%3u,%3u,%3u,%3u,%3u, %s },  ",
                 n->res.params.windowLog, n->res.params.chainLog, n->res.params.hashLog, n->res.params.searchLog, n->res.params.searchLength,
@@ -1045,6 +1042,12 @@ static void BMK_printWinnerOpt(FILE* f, const U32 cLevel, const BMK_result_t res
             (double)srcSize / result.cSize, (double)result.cSpeed / (1 MB), (double)result.dSpeed / (1 MB));
 
     }
+
+#if 0
+    if(BMK_timeSpan(g_time) > g_grillDuration_s) {
+        exit(0);
+    }
+#endif
 }
 
 static void BMK_printWinners2(FILE* f, const winnerInfo_t* winners, size_t srcSize)
@@ -2097,7 +2100,7 @@ static ZSTD_compressionParameters maskParams(ZSTD_compressionParameters base, ZS
  * cLevel - compression level to exceed (all solutions must be > lvl in cSpeed + ratio)
  */
 
-#define MAX_TRIES 5
+static int g_maxTries = 5;
 #define TRY_DECAY 1
 
 static int optimizeForSize(const char* const * const fileNamesTable, const size_t nbFiles, const char* dictFileName, constraint_t target, ZSTD_compressionParameters paramTarget, int cLevel)
@@ -2185,20 +2188,21 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_
     }
 
     /* use level'ing mode instead of normal target mode */
+    /* Should lvl be parameter-masked here? */
     if(g_optmode) {
         winner.params = ZSTD_getCParams(cLevel, maxBlockSize, ctx.dictSize);
         if(BMK_benchParam(&winner.result, buf, ctx, winner.params)) {
             ret = 3;
             goto _cleanUp;
         }
-
-        target.cSpeed = (U32)winner.result.cSpeed;
-
-        g_targetConstraints = target;
-
+ 
         g_lvltarget = winner.result; 
-        g_lvltarget.cSpeed *= ((double)(g_strictness) / 100);
-        g_lvltarget.cSize /= ((double)(g_strictness) / 100);
+        g_lvltarget.cSpeed *= ((double)g_strictness) / 100;
+        g_lvltarget.dSpeed *= ((double)g_strictness) / 100;
+        g_lvltarget.cSize /= ((double)g_strictness) / 100;
+
+        target.cSpeed = (U32)g_lvltarget.cSpeed;  
+        target.dSpeed = (U32)g_lvltarget.dSpeed; //See if this is worth
 
         BMK_printWinnerOpt(stdout, cLevel, winner.result, winner.params, target, buf.srcSize);
     }
@@ -2241,7 +2245,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_
                     }
 
                     /* if the current params are too slow, just stop. */
-                    if(target.cSpeed > candidate.cSpeed * 2) { break; }
+                    if(target.cSpeed > candidate.cSpeed * 3 / 2) { break; }
                 }
             }
         }
@@ -2254,7 +2258,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_
             int bestStrategy = (int)winner.params.strategy;
             if(paramTarget.strategy == 0) {
                 int st = (int)winner.params.strategy;
-                int tries = MAX_TRIES;
+                int tries = g_maxTries;
 
                 { 
                     /* one iterations of hill climbing with the level-defined parameters. */
@@ -2269,12 +2273,13 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_
                 while(st && tries > 0) {
                     winnerInfo_t wc;
                     DEBUGOUTPUT("StrategySwitch: %s\n", g_stratName[st]);
+                    
                     wc = optimizeFixedStrategy(buf, ctx, target, paramTarget, 
                         st, varArray, varLen, allMT[st], tries);
 
                     if(compareResultLT(winner.result, wc.result, target, buf.srcSize)) {
                         winner = wc;
-                        tries = MAX_TRIES;
+                        tries = g_maxTries;
                         bestStrategy = st;
                     } else {
                         st = nextStrategy(st, bestStrategy);
@@ -2283,7 +2288,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_
                 }
             } else {
                 winner = optimizeFixedStrategy(buf, ctx, target, paramTarget, paramTarget.strategy, 
-                    varArray, varLen, allMT[paramTarget.strategy], 10);
+                    varArray, varLen, allMT[paramTarget.strategy], g_maxTries);
             }
 
         }
@@ -2402,17 +2407,17 @@ int main(int argc, const char** argv)
 
     ZSTD_compressionParameters paramTarget = { 0, 0, 0, 0, 0, 0, 0 };
 
-    assert(argc>=1);   /* for exename */
-
     g_time = UTIL_getTime();
 
+    assert(argc>=1);   /* for exename */
+
     /* Welcome message */
     DISPLAY(WELCOME_MESSAGE);
 
     for(i=1; i
Date: Mon, 30 Jul 2018 17:42:46 -0700
Subject: [PATCH 153/372] Update fulltable to use same interface

Add seperateFiles flag
---
 tests/README.md    |   1 +
 tests/paramgrill.c | 389 ++++++++++++++++++++++++---------------------
 2 files changed, 207 insertions(+), 183 deletions(-)

diff --git a/tests/README.md b/tests/README.md
index 946f890b1..736916936 100644
--- a/tests/README.md
+++ b/tests/README.md
@@ -125,6 +125,7 @@ Full list of arguments
  -t#          : Caps runtime of operation in seconds (default : 99999 seconds (about 27 hours )) 
  -v           : Prints Benchmarking output
  -D           : Next argument dictionary file
+ -s           : Benchmark all files separately
 
 ```
  Any inputs afterwards are treated as files to benchmark.
diff --git a/tests/paramgrill.c b/tests/paramgrill.c
index 7e86c2e9e..cba97ebbb 100644
--- a/tests/paramgrill.c
+++ b/tests/paramgrill.c
@@ -96,7 +96,7 @@ typedef enum {
 static const int rangetable[NUM_PARAMS] = { WLOG_RANGE, CLOG_RANGE, HLOG_RANGE, SLOG_RANGE, SLEN_RANGE, TLEN_RANGE };
 static const U32 tlen_table[TLEN_RANGE] = { 0, 1, 2, 4, 6, 8, 12, 16, 24, 32, 48, 64, 96, 128, 256, 512, 999 };
 /*-************************************
-*  Benchmark Parameters
+*  Benchmark Parameters/Global Variables
 **************************************/
 
 typedef BYTE U8;
@@ -110,7 +110,7 @@ static U32 g_singleRun = 0;
 static U32 g_optimizer = 0;
 static U32 g_target = 0;
 static U32 g_noSeed = 0;
-static ZSTD_compressionParameters g_params = { 0, 0, 0, 0, 0, 0, ZSTD_greedy };
+static ZSTD_compressionParameters g_params;
 static UTIL_time_t g_time; /* to be used to compare solution finding speeds to compare to original */
 
 
@@ -150,6 +150,14 @@ void BMK_SetNbIterations(int nbLoops)
     DISPLAY("- %u iterations -\n", g_nbIterations);
 }
 
+/*
+ * Additional Global Variables (Defined Above Use)
+ * g_stratName
+ * g_level_constraint
+ * g_alreadyTested
+ * g_maxTries
+ */
+
 /*-*******************************************************
 *  Private functions
 *********************************************************/
@@ -335,17 +343,6 @@ const char* g_stratName[ZSTD_btultra+1] = {
                 "ZSTD_greedy  ", "ZSTD_lazy    ", "ZSTD_lazy2   ",
                 "ZSTD_btlazy2 ", "ZSTD_btopt   ", "ZSTD_btultra "};
 
-/* benchParam but only takes in one input buffer. */
-static int
-BMK_benchParam1(BMK_result_t* resultPtr,
-               const void* srcBuffer, size_t srcSize,
-               const ZSTD_compressionParameters cParams) {
-
-    BMK_return_t res = BMK_benchMem(srcBuffer,srcSize, &srcSize, 1, 0, &cParams, NULL, 0, 0, "File");
-    *resultPtr = res.result;
-    return res.error;
-}
-
 static ZSTD_compressionParameters emptyParams(void) {
     ZSTD_compressionParameters p = { 0, 0, 0, 0, 0, 0, (ZSTD_strategy)0 };
     return p;
@@ -381,14 +378,6 @@ typedef struct {
     ZSTD_DCtx* dctx;
 } contexts_t;
 
-static int BMK_benchParam(BMK_result_t* resultPtr,
-                const buffers_t buf, const contexts_t ctx,
-                const ZSTD_compressionParameters cParams) {
-    BMK_return_t res = BMK_benchMem(buf.srcPtrs[0], buf.srcSize, buf.srcSizes, (unsigned)buf.nbBlocks, 0, &cParams, ctx.dictBuffer, ctx.dictSize, 0, "Files");
-    *resultPtr = res.result;
-    return res.error;
-}
-
 /*-*******************************************************
 *  From Paramgrill
 *********************************************************/
@@ -542,7 +531,7 @@ static int createBuffers(buffers_t* buff, const char* const * const fileNamesTab
     size_t n;
     U64 const totalSizeToLoad = UTIL_getTotalFileSize(fileNamesTable, (U32)nbFiles);
     size_t benchedSize = MIN(BMK_findMaxMem(totalSizeToLoad * 3) / 3, totalSizeToLoad);
-    const size_t blockSize = g_blockSize ? g_blockSize : totalSizeToLoad; //(largest fileSize or total fileSize)
+    const size_t blockSize = g_blockSize ? g_blockSize : totalSizeToLoad;
     U32 const maxNbBlocks = (U32) ((totalSizeToLoad + (blockSize-1)) / blockSize) + (U32)nbFiles;
     U32 blockNb = 0;
 
@@ -830,6 +819,14 @@ static BMK_return_t BMK_benchMemInvertible(const buffers_t buf, const contexts_t
     return results;
 }
 
+static int BMK_benchParam(BMK_result_t* resultPtr,
+                buffers_t buf, contexts_t ctx,
+                const ZSTD_compressionParameters cParams) {
+    BMK_return_t res = BMK_benchMemInvertible(buf, ctx, 0, &cParams, BMK_both, BMK_timeMode, 3);
+    *resultPtr = res.result;
+    return res.error;
+}
+
 /* comparison function: */
 /* strictly better, strictly worse, equal, speed-side adv, size-side adv */
 //Maybe use compress_only for benchmark first run?
@@ -911,7 +908,7 @@ static int insertWinner(winnerInfo_t w, constraint_t targetConstraints) {
 
     }
 
-    //assert(cur_node->next == NULL)
+    assert(cur_node->next == NULL);
     switch(speedSizeCompare(r, cur_node->res.result)) {
         case BETTER_RESULT:
         {
@@ -995,7 +992,7 @@ static void BMK_printWinnerOpt(FILE* f, const U32 cLevel, const BMK_result_t res
         }
     }  
 
-    //prints out tradeoff table if using lvl
+    //prints out tradeoff table if using lvloptimize
     if(g_optmode && g_optimizer) {
         winnerInfo_t w;
         winner_ll_node* n;
@@ -1042,12 +1039,6 @@ static void BMK_printWinnerOpt(FILE* f, const U32 cLevel, const BMK_result_t res
             (double)srcSize / result.cSize, (double)result.cSpeed / (1 MB), (double)result.dSpeed / (1 MB));
 
     }
-
-#if 0
-    if(BMK_timeSpan(g_time) > g_grillDuration_s) {
-        exit(0);
-    }
-#endif
 }
 
 static void BMK_printWinners2(FILE* f, const winnerInfo_t* winners, size_t srcSize)
@@ -1099,14 +1090,14 @@ static void BMK_init_level_constraints(int bytePerSec_level1)
     }   }
 }
 
-static int BMK_seed(winnerInfo_t* winners, const ZSTD_compressionParameters params,
-              const void* srcBuffer, size_t srcSize)
+static int BMK_seed(winnerInfo_t* winners, const ZSTD_compressionParameters params, 
+                    buffers_t buf, contexts_t ctx)
 {
     BMK_result_t testResult;
     int better = 0;
     int cLevel;
 
-    BMK_benchParam1(&testResult, srcBuffer, srcSize, params);
+    BMK_benchParam(&testResult, buf, ctx, params);
 
 
     for (cLevel = 1; cLevel <= NB_LEVELS_TRACKED; cLevel++) {
@@ -1122,15 +1113,15 @@ static int BMK_seed(winnerInfo_t* winners, const ZSTD_compressionParameters para
             /* first solution for this cLevel */
             winners[cLevel].result = testResult;
             winners[cLevel].params = params;
-            BMK_printWinner(stdout, cLevel, testResult, params, srcSize);
+            BMK_printWinner(stdout, cLevel, testResult, params, buf.srcSize);
             better = 1;
             continue;
         }
 
         if ((double)testResult.cSize <= ((double)winners[cLevel].result.cSize * (1. + (0.02 / cLevel))) ) {
             /* Validate solution is "good enough" */
-            double W_ratio = (double)srcSize / testResult.cSize;
-            double O_ratio = (double)srcSize / winners[cLevel].result.cSize;
+            double W_ratio = (double)buf.srcSize / testResult.cSize;
+            double O_ratio = (double)buf.srcSize / winners[cLevel].result.cSize;
             double W_ratioNote = log (W_ratio);
             double O_ratioNote = log (O_ratio);
             size_t W_DMemUsed = (1 << params.windowLog) + (16 KB);
@@ -1187,7 +1178,7 @@ static int BMK_seed(winnerInfo_t* winners, const ZSTD_compressionParameters para
 
             winners[cLevel].result = testResult;
             winners[cLevel].params = params;
-            BMK_printWinner(stdout, cLevel, testResult, params, srcSize);
+            BMK_printWinner(stdout, cLevel, testResult, params, buf.srcSize);
 
             better = 1;
     }   }
@@ -1315,6 +1306,26 @@ static void paramVariation(ZSTD_compressionParameters* ptr, const varInds_t* var
     *ptr = p;
 }
 
+/* maybe put strategy back in */
+static void paramVariationWithStrategy(ZSTD_compressionParameters* ptr, const varInds_t* varyParams, const int varyLen, const U32 nbChanges)
+{
+    ZSTD_compressionParameters p;
+    U32 validated = 0;
+    while (!validated) {
+        U32 i;
+        p = *ptr;
+        for (i = 0 ; i < nbChanges ; i++) {
+            const U32 changeID = FUZ_rand(&g_rand) % ((varyLen + 1) << 1);
+            if(changeID < (U32)(varyLen << 1)) {
+                paramVaryOnce(varyParams[changeID >> 1], ((changeID & 1) << 1) - 1, &p);
+            } else {
+                p.strategy += ((FUZ_rand(&g_rand) % 2) << 1) - 1; /* +/- 1 */
+            }
+        }
+        validated = !ZSTD_isError(ZSTD_checkCParams(p));
+    }
+    *ptr = p;
+}
 /* length of memo table given free variables */
 static size_t memoTableLen(const varInds_t* varyParams, const int varyLen) {
     size_t arrayLen = 1;
@@ -1463,6 +1474,17 @@ static U8** createMemoTableArray(ZSTD_compressionParameters paramConstraints, co
     return mtAll;
 }
 
+static ZSTD_compressionParameters maskParams(ZSTD_compressionParameters base, ZSTD_compressionParameters mask) {
+    base.windowLog = mask.windowLog ? mask.windowLog : base.windowLog;
+    base.chainLog = mask.chainLog ? mask.chainLog : base.chainLog;
+    base.hashLog = mask.hashLog ? mask.hashLog : base.hashLog;
+    base.searchLog = mask.searchLog ? mask.searchLog : base.searchLog;
+    base.searchLength = mask.searchLength ? mask.searchLength : base.searchLength;
+    base.targetLength = mask.targetLength ? mask.targetLength : base.targetLength;
+    base.strategy = mask.strategy ? mask.strategy : base.strategy;
+    return base;
+}
+
 #define PARAMTABLELOG   25
 #define PARAMTABLESIZE (1< g_maxNbVariations) break;
-        paramVariation(&p, unconstrained, 7, 4);
+        paramVariationWithStrategy(&p, unconstrained, NUM_PARAMS, 4);
 
         /* exclude faster if already played params */
         if (FUZ_rand(&g_rand) & ((1 << *NB_TESTS_PLAYED(p))-1))
@@ -1500,11 +1522,11 @@ static void playAround(FILE* f, winnerInfo_t* winners,
         /* test */
         b = NB_TESTS_PLAYED(p);
         (*b)++;
-        if (!BMK_seed(winners, p, srcBuffer, srcSize)) continue;
+        if (!BMK_seed(winners, p, buf, ctx)) continue;
 
         /* improvement found => search more */
-        BMK_printWinners(f, winners, srcSize);
-        playAround(f, winners, p, srcBuffer, srcSize);
+        BMK_printWinners(f, winners, buf.srcSize);
+        playAround(f, winners, p, buf, ctx);
     }
 
 }
@@ -1551,35 +1573,24 @@ static void randomConstrainedParams(ZSTD_compressionParameters* pc, varInds_t* v
 
 static void BMK_selectRandomStart(
                        FILE* f, winnerInfo_t* winners,
-                       const void* srcBuffer, size_t srcSize)
+                       buffers_t buf, contexts_t ctx)
 {
     U32 const id = FUZ_rand(&g_rand) % (NB_LEVELS_TRACKED+1);
     if ((id==0) || (winners[id].params.windowLog==0)) {
         /* use some random entry */
-        ZSTD_compressionParameters const p = ZSTD_adjustCParams(randomParams(), srcSize, 0);
-        playAround(f, winners, p, srcBuffer, srcSize);
+        ZSTD_compressionParameters const p = ZSTD_adjustCParams(randomParams(), buf.srcSize, 0);
+        playAround(f, winners, p, buf, ctx);
     } else {
-        playAround(f, winners, winners[id].params, srcBuffer, srcSize);
+        playAround(f, winners, winners[id].params, buf, ctx);
     }
 }
 
-static void BMK_benchOnce(const void* srcBuffer, size_t srcSize)
-{
-    BMK_result_t testResult;
-    g_params = ZSTD_adjustCParams(g_params, srcSize, 0);
-    BMK_benchParam1(&testResult, srcBuffer, srcSize, g_params);
-    DISPLAY("Compression Ratio: %.3f  Compress Speed: %.1f MB/s Decompress Speed: %.1f MB/s\n", (double)srcSize / testResult.cSize, 
-        (double)testResult.cSpeed / 1000000, (double)testResult.dSpeed / 1000000);
-    return;
-}
-
-static void BMK_benchFullTable(const void* srcBuffer, size_t srcSize)
+static void BMK_benchFullTable(buffers_t buf, contexts_t ctx, const size_t maxBlockSize) 
 {
     ZSTD_compressionParameters params;
     winnerInfo_t winners[NB_LEVELS_TRACKED+1];
     const char* const rfName = "grillResults.txt";
     FILE* const f = fopen(rfName, "w");
-    const size_t blockSize = g_blockSize ? g_blockSize : srcSize;   /* cut by block or not ? */
 
     /* init */
     assert(g_singleRun==0);
@@ -1590,9 +1601,9 @@ static void BMK_benchFullTable(const void* srcBuffer, size_t srcSize)
         BMK_init_level_constraints(g_target * (1 MB));
     } else {
         /* baseline config for level 1 */
-        ZSTD_compressionParameters const l1params = ZSTD_getCParams(1, blockSize, 0);
+        ZSTD_compressionParameters const l1params = ZSTD_getCParams(1, maxBlockSize, ctx.dictSize); //is dictionary ever even useful here? 
         BMK_result_t testResult;
-        BMK_benchParam1(&testResult, srcBuffer, srcSize, l1params);
+        BMK_benchParam(&testResult, buf, ctx, l1params);
         BMK_init_level_constraints((int)((testResult.cSpeed * 31) / 32));
     }
 
@@ -1600,61 +1611,124 @@ static void BMK_benchFullTable(const void* srcBuffer, size_t srcSize)
     {   const int maxSeeds = g_noSeed ? 1 : ZSTD_maxCLevel();
         int i;
         for (i=0; i<=maxSeeds; i++) {
-            params = ZSTD_getCParams(i, blockSize, 0);
-            BMK_seed(winners, params, srcBuffer, srcSize);
+            params = ZSTD_getCParams(i, maxBlockSize, 0);
+            BMK_seed(winners, params, buf, ctx);
     }   }
-    BMK_printWinners(f, winners, srcSize);
+    BMK_printWinners(f, winners, buf.srcSize);
 
     /* start tests */
     {   const time_t grillStart = time(NULL);
         do {
-            BMK_selectRandomStart(f, winners, srcBuffer, srcSize);
+            BMK_selectRandomStart(f, winners, buf, ctx);
         } while (BMK_timeSpan(grillStart) < g_grillDuration_s);
     }
 
     /* end summary */
-    BMK_printWinners(f, winners, srcSize);
+    BMK_printWinners(f, winners, buf.srcSize);
     DISPLAY("grillParams operations completed \n");
 
     /* clean up*/
     fclose(f);
 }
 
-static void BMK_benchMemInit(const void* srcBuffer, size_t srcSize)
-{
-    if (g_singleRun)
-        return BMK_benchOnce(srcBuffer, srcSize);
-    else
-        return BMK_benchFullTable(srcBuffer, srcSize);
-}
-
-
 static int benchSample(void)
 {
     const char* const name = "Sample 10MB";
     size_t const benchedSize = 10 MB;
+    U32 blockSize = g_blockSize ? g_blockSize : benchedSize;
+    U32 const maxNbBlocks = (U32) ((benchedSize + (blockSize-1)) / blockSize) + 1;
+    size_t splitSize = 0;
 
-    void* origBuff = malloc(benchedSize);
-    if (!origBuff) { perror("not enough memory"); return 12; }
+    buffers_t buf;
+    contexts_t ctx;
 
-    /* Fill buffer */
-    RDG_genBuffer(origBuff, benchedSize, g_compressibility, 0.0, 0);
+    buf.srcPtrs = (const void**)calloc(maxNbBlocks, sizeof(void*));
+    buf.dstPtrs = (void**)calloc(maxNbBlocks, sizeof(void*));
+    buf.resPtrs = (void**)calloc(maxNbBlocks, sizeof(void*));
+    buf.srcSizes = (size_t*)malloc(maxNbBlocks * sizeof(size_t));
+    buf.dstSizes = (size_t*)malloc(maxNbBlocks * sizeof(size_t));
+    buf.dstCapacities = (size_t*)malloc(maxNbBlocks * sizeof(size_t));
+    buf.resSizes = (size_t*)malloc(maxNbBlocks * sizeof(size_t));
+    buf.srcSize = benchedSize;
+
+    if(!buf.srcPtrs || !buf.dstPtrs || !buf.resPtrs || !buf.srcSizes || !buf.dstSizes || !buf.dstCapacities || !buf.resSizes) {
+        DISPLAY("Allocation Error\n");
+        freeBuffers(buf);
+        return 1;
+    } 
+
+    buf.srcBuffer = malloc(benchedSize);
+    buf.srcPtrs[0] = (const void*)buf.srcBuffer;
+    buf.dstPtrs[0] = malloc(ZSTD_compressBound(benchedSize) + 1024 * maxNbBlocks);
+    buf.resPtrs[0] = malloc(benchedSize);
+
+    if(!buf.srcPtrs[0] || !buf.dstPtrs[0] || !buf.resPtrs[0]) {
+        DISPLAY("Allocation Error\n");
+        freeBuffers(buf);
+        return 1;
+    }
+
+
+    splitSize = MIN(benchedSize, blockSize);
+    buf.srcSizes[0] = splitSize;
+    buf.dstCapacities[0] = ZSTD_compressBound(splitSize);
+    buf.resSizes[0] = splitSize;
+
+    for(buf.nbBlocks = 1; splitSize < benchedSize; buf.nbBlocks++) {
+        const size_t i = buf.nbBlocks;
+        const size_t nextBlockSize = MIN(benchedSize - splitSize, blockSize);
+        buf.srcSizes[i] = nextBlockSize;
+        buf.dstCapacities[i] = ZSTD_compressBound(nextBlockSize);
+        buf.resSizes[i] = nextBlockSize;
+        buf.srcPtrs[i] = (const void*)(((const char*)buf.srcPtrs[i-1]) + buf.srcSizes[i-1]);
+        buf.dstPtrs[i] = (void*)(((char*)buf.dstPtrs[i-1]) + buf.dstSizes[i-1]);
+        buf.resPtrs[i] = (void*)(((char*)buf.resPtrs[i-1]) + buf.resSizes[i-1]);
+        splitSize += nextBlockSize;
+    }
+
+    if(createContexts(&ctx, NULL)) {
+        DISPLAY("Context Creation Error\n");
+        freeBuffers(buf);
+        return 1;
+    }
+
+    RDG_genBuffer(buf.srcBuffer, benchedSize, g_compressibility, 0.0, 0);
 
     /* bench */
     DISPLAY("\r%79s\r", "");
     DISPLAY("using %s %i%%: \n", name, (int)(g_compressibility*100));
-    BMK_benchMemInit(origBuff, benchedSize);
 
-    free(origBuff);
+    BMK_benchFullTable(buf, ctx, MIN(blockSize, benchedSize));
+
+    freeBuffers(buf);
+    freeContexts(ctx);
+
     return 0;
 }
 
 
-static int benchOnce(const char** fileNamesTable, int nbFiles, const char* dictFileName) {
+static int benchOnce(buffers_t buf, contexts_t ctx) {
+    BMK_result_t testResult;
+
+    if(BMK_benchParam(&testResult, buf, ctx, g_params)) {
+        DISPLAY("Error during benchmarking\n");
+        return 1;
+    }
+
+    DISPLAY("Compression Ratio: %.3f  Compress Speed: %.1f MB/s Decompress Speed: %.1f MB/s\n", (double)buf.srcSize / testResult.cSize, 
+        (double)testResult.cSpeed / (1 MB), (double)testResult.dSpeed / (1 MB));
+    return 0;
+}
+
+/* benchFiles() :
+ * note: while this function takes a table of filenames,
+ * in practice, only the first filename will be used */
+int benchFiles(const char** fileNamesTable, int nbFiles, const char* dictFileName, int cLevel)
+{
     buffers_t buf;
     contexts_t ctx;
-    BMK_result_t testResult;
     size_t maxBlockSize = 0, i;
+    int ret = 0;
 
     if(createBuffers(&buf, fileNamesTable, nbFiles)) {
         DISPLAY("unable to load files\n");
@@ -1671,86 +1745,24 @@ static int benchOnce(const char** fileNamesTable, int nbFiles, const char* dictF
         maxBlockSize = MAX(maxBlockSize, buf.srcSizes[i]);
     }
 
-    g_params = ZSTD_adjustCParams(g_params, maxBlockSize, 0);
-
-    if(BMK_benchParam(&testResult, buf, ctx, g_params)) {
-        DISPLAY("Error during benchmarking\n");
-        freeBuffers(buf);
-        freeContexts(ctx);
-        return 3;
+    DISPLAY("\r%79s\r", "");
+    if(nbFiles == 1) {
+        DISPLAY("using %s : \n", fileNamesTable[0]);
+    } else {
+        DISPLAY("using %d Files : \n", nbFiles);
     }
 
-    DISPLAY("Compression Ratio: %.3f  Compress Speed: %.1f MB/s Decompress Speed: %.1f MB/s\n", (double)buf.srcSize / testResult.cSize, 
-        (double)testResult.cSpeed / (1 MB), (double)testResult.dSpeed / (1 MB));
+    g_params = ZSTD_adjustCParams(maskParams(ZSTD_getCParams(cLevel, maxBlockSize, ctx.dictSize), g_params), maxBlockSize, ctx.dictSize);
+
+    if(g_singleRun) {
+        ret = benchOnce(buf, ctx);
+    } else {
+        BMK_benchFullTable(buf, ctx, maxBlockSize);
+    }
 
     freeBuffers(buf);
     freeContexts(ctx);
-    return 0;
-}
-
-/* benchFiles() :
- * note: while this function takes a table of filenames,
- * in practice, only the first filename will be used */
-//TODO: dictionaries still not supported in fullTable mode
-int benchFiles(const char** fileNamesTable, int nbFiles, const char* dictFileName)
-{
-    int fileIdx=0;
-
-    if(g_singleRun) {
-        return benchOnce(fileNamesTable, nbFiles, dictFileName);
-    }
-
-    /* Loop for each file */
-    while (fileIdx inFileSize) benchedSize = (size_t)inFileSize;
-        if (benchedSize < inFileSize)
-            DISPLAY("Not enough memory for '%s' full size; testing %i MB only...\n", inFileName, (int)(benchedSize>>20));
-        origBuff = malloc(benchedSize);
-        if (origBuff==NULL) {
-            DISPLAY("\nError: not enough memory!\n");
-            fclose(inFile);
-            return 12;
-        }
-
-        /* Fill input buffer */
-        DISPLAY("Loading %s...       \r", inFileName);
-        {   size_t const readSize = fread(origBuff, 1, benchedSize, inFile);
-            fclose(inFile);
-            if(readSize != benchedSize) {
-                DISPLAY("\nError: problem reading file '%s' !!    \n", inFileName);
-                free(origBuff);
-                return 13;
-        }   }
-
-        /* bench */
-        DISPLAY("\r%79s\r", "");
-        DISPLAY("using %s : \n", inFileName);
-        BMK_benchMemInit(origBuff, benchedSize);
-
-        /* clean */
-        free(origBuff);
-    }
-
-    return 0;
+    return ret;
 }
 
 /* Benchmarking which stops when we are sufficiently sure the solution is infeasible / worse than the winner */
@@ -2074,17 +2086,6 @@ static int nextStrategy(const int currentStrategy, const int bestStrategy) {
     }
 }
 
-static ZSTD_compressionParameters maskParams(ZSTD_compressionParameters base, ZSTD_compressionParameters mask) {
-    base.windowLog = mask.windowLog ? mask.windowLog : base.windowLog;
-    base.chainLog = mask.chainLog ? mask.chainLog : base.chainLog;
-    base.hashLog = mask.hashLog ? mask.hashLog : base.hashLog;
-    base.searchLog = mask.searchLog ? mask.searchLog : base.searchLog;
-    base.searchLength = mask.searchLength ? mask.searchLength : base.searchLength;
-    base.targetLength = mask.targetLength ? mask.targetLength : base.targetLength;
-    base.strategy = mask.strategy ? mask.strategy : base.strategy;
-    return base;
-}
-
 /* experiment with playing with this and decay value */
 
 /* main fn called when using --optimize */
@@ -2115,6 +2116,8 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_
     contexts_t ctx;
     buffers_t buf;
 
+    g_time = UTIL_getTime();
+
     /* Init */
     if(!cParamValid(paramTarget)) {
         return 1;
@@ -2202,7 +2205,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_
         g_lvltarget.cSize /= ((double)g_strictness) / 100;
 
         target.cSpeed = (U32)g_lvltarget.cSpeed;  
-        target.dSpeed = (U32)g_lvltarget.dSpeed; //See if this is worth
+        target.dSpeed = (U32)g_lvltarget.dSpeed; //See if this is reasonable. 
 
         BMK_printWinnerOpt(stdout, cLevel, winner.result, winner.params, target, buf.srcSize);
     }
@@ -2214,6 +2217,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_
     } else {
         DISPLAY("optimizing for %lu Files", (unsigned long)nbFiles);
     }
+
     if(target.cSpeed != 0) { DISPLAY(" - limit compression speed %u MB/s", target.cSpeed >> 20); }
     if(target.dSpeed != 0) { DISPLAY(" - limit decompression speed %u MB/s", target.dSpeed >> 20); }
     if(target.cMem != (U32)-1) { DISPLAY(" - limit memory %u MB", target.cMem >> 20); }
@@ -2369,6 +2373,7 @@ static int usage_advanced(void)
     DISPLAY( " -t#          : Caps runtime of operation in seconds (default : %u seconds (%.1f hours)) \n", (U32)g_grillDuration_s, g_grillDuration_s / 3600);
     DISPLAY( " -v           : Prints Benchmarking output\n");
     DISPLAY( " -D           : Next argument dictionary file\n");
+    DISPLAY( " -s           : Seperate Files\n");
     return 0;
 }
 
@@ -2401,13 +2406,13 @@ int main(int argc, const char** argv)
     const char* input_filename = NULL;
     const char* dictFileName = NULL;
     U32 main_pause = 0;
-    int optimizerCLevel = 0;
+    int cLevel = 0;
+    int seperateFiles = 0;
 
     constraint_t target = { 0, 0, (U32)-1 }; 
 
-    ZSTD_compressionParameters paramTarget = { 0, 0, 0, 0, 0, 0, 0 };
-
-    g_time = UTIL_getTime();
+    ZSTD_compressionParameters paramTarget = emptyParams();
+    g_params = emptyParams();
 
     assert(argc>=1);   /* for exename */
 
@@ -2430,11 +2435,11 @@ int main(int argc, const char** argv)
                 PARSE_SUB_ARGS("compressionSpeed=" ,  "cSpeed=", target.cSpeed);
                 PARSE_SUB_ARGS("decompressionSpeed=", "dSpeed=", target.dSpeed); 
                 PARSE_SUB_ARGS("compressionMemory=" , "cMem=", target.cMem);
-                PARSE_SUB_ARGS("level=", "lvl=", optimizerCLevel);
+                PARSE_SUB_ARGS("level=", "lvl=", cLevel);
                 PARSE_SUB_ARGS("strict=", "stc=", g_strictness);
                 PARSE_SUB_ARGS("preferSpeed=", "prfSpd=", g_speedMultiplier);
                 PARSE_SUB_ARGS("preferRatio=", "prfRto=", g_ratioMultiplier);
-                PARSE_SUB_ARGS("maxTries", "tries", g_maxTries);
+                PARSE_SUB_ARGS("maxTries=", "tries=", g_maxTries);
 
                 DISPLAY("invalid optimization parameter \n");
                 return 1;
@@ -2448,10 +2453,11 @@ int main(int argc, const char** argv)
         } else if (longCommandWArg(&argument, "--zstd=")) {
         /* Decode command (note : aggregated commands are allowed) */
             g_singleRun = 1;
-            g_params = ZSTD_getCParams(2, g_blockSize, 0);
+            cLevel = 2;
             for ( ; ;) {
                 PARSE_CPARAMS(g_params)
-                if (longCommandWArg(&argument, "level=") || longCommandWArg(&argument, "lvl=")) { g_params = ZSTD_getCParams(readU32FromChar(&argument), g_blockSize, 0); if (argument[0]==',') { argument++; continue; } else break; }
+                if (longCommandWArg(&argument, "level=") || longCommandWArg(&argument, "lvl=")) { cLevel = readU32FromChar(&argument); g_params = emptyParams(); if (argument[0]==',') { argument++; continue; } else break; }
+
                 DISPLAY("invalid compression parameter \n");
                 return 1;
             }
@@ -2528,8 +2534,8 @@ int main(int argc, const char** argv)
                             continue;
                         case 'L':
                             {   argument++;
-                                int const cLevel = readU32FromChar(&argument);
-                                g_params = ZSTD_getCParams(cLevel, g_blockSize, 0);
+                                cLevel = readU32FromChar(&argument);
+                                g_params = emptyParams();
                                 continue;
                             }
                         default : ;
@@ -2558,6 +2564,10 @@ int main(int argc, const char** argv)
                     g_grillDuration_s = (double)readU32FromChar(&argument);
                     break;
 
+                case 's':
+                    seperateFiles = 1;
+                    break;
+
                 /* load dictionary file (only applicable for optimizer rn) */
                 case 'D':
                     if(i == argc - 1) { /* last argument, return error. */ 
@@ -2588,11 +2598,24 @@ int main(int argc, const char** argv)
             result = benchSample();
         }
     } else {
-        if (g_optimizer) {
-            result = optimizeForSize(argv+filenamesStart, argc-filenamesStart, dictFileName, target, paramTarget, optimizerCLevel);
+        if(seperateFiles) {
+            for(i = 0; i < argc - filenamesStart; i++) {
+                if (g_optimizer) {
+                    result = optimizeForSize(argv+filenamesStart + i, 1, dictFileName, target, paramTarget, cLevel);
+                    if(result) { DISPLAY("Error on File %d", i); return result; }
+                } else {
+                    result = benchFiles(argv+filenamesStart + i, 1, dictFileName, cLevel);
+                    if(result) { DISPLAY("Error on File %d", i); return result; }
+                }
+            }
         } else {
-            result = benchFiles(argv+filenamesStart, argc-filenamesStart, dictFileName);
-    }   }
+            if (g_optimizer) {
+                result = optimizeForSize(argv+filenamesStart, argc-filenamesStart, dictFileName, target, paramTarget, cLevel);
+            } else {
+                result = benchFiles(argv+filenamesStart, argc-filenamesStart, dictFileName, cLevel);
+            }
+        }   
+    }
 
     if (main_pause) { int unused; printf("press enter...\n"); unused = getchar(); (void)unused; }
 

From 3b36fe5c68952dd03556ab04ff50dcbe0f1c9ec4 Mon Sep 17 00:00:00 2001
From: George Lu 
Date: Tue, 31 Jul 2018 11:13:44 -0700
Subject: [PATCH 154/372] strategy switching

---
 tests/paramgrill.c | 124 +++++++++++++++++++++++----------------------
 1 file changed, 64 insertions(+), 60 deletions(-)

diff --git a/tests/paramgrill.c b/tests/paramgrill.c
index cba97ebbb..955032e84 100644
--- a/tests/paramgrill.c
+++ b/tests/paramgrill.c
@@ -73,10 +73,11 @@ typedef enum {
     hlog_ind = 2,
     slog_ind = 3,
     slen_ind = 4,
-    tlen_ind = 5
+    tlen_ind = 5,
+    strt_ind = 6
 } varInds_t;
 
-#define NUM_PARAMS 6
+#define NUM_PARAMS 7
 /* just don't use strategy as a param. */ 
 
 #undef ZSTD_WINDOWLOG_MAX
@@ -91,9 +92,10 @@ typedef enum {
 #define SLOG_RANGE (ZSTD_SEARCHLOG_MAX - ZSTD_SEARCHLOG_MIN + 1)
 #define SLEN_RANGE (ZSTD_SEARCHLENGTH_MAX - ZSTD_SEARCHLENGTH_MIN + 1)
 #define TLEN_RANGE 17
+#define STRT_RANGE (ZSTD_btultra - ZSTD_fast + 1)
 /* TLEN_RANGE picked manually */
 
-static const int rangetable[NUM_PARAMS] = { WLOG_RANGE, CLOG_RANGE, HLOG_RANGE, SLOG_RANGE, SLEN_RANGE, TLEN_RANGE };
+static const int rangetable[NUM_PARAMS] = { WLOG_RANGE, CLOG_RANGE, HLOG_RANGE, SLOG_RANGE, SLEN_RANGE, TLEN_RANGE, STRT_RANGE };
 static const U32 tlen_table[TLEN_RANGE] = { 0, 1, 2, 4, 6, 8, 12, 16, 24, 32, 48, 64, 96, 128, 256, 512, 999 };
 /*-************************************
 *  Benchmark Parameters/Global Variables
@@ -1213,8 +1215,10 @@ static int sanitizeVarArray(varInds_t* varNew, const int varLength, const varInd
     int i, j = 0;
     for(i = 0; i < varLength; i++) {
         if( !((varArray[i] == clog_ind && strat == ZSTD_fast)
+            || (varArray[i] == slog_ind && strat == ZSTD_fast)
             || (varArray[i] == slog_ind && strat == ZSTD_dfast) 
-            || (varArray[i] == tlen_ind && strat != ZSTD_btopt && strat != ZSTD_btultra && strat != ZSTD_fast))) {
+            || (varArray[i] == tlen_ind && strat != ZSTD_btopt && strat != ZSTD_btultra && strat != ZSTD_fast)
+            /* || varArray[i] == strt_ind */ )) {
             varNew[j] = varArray[i];
             j++;
         }
@@ -1251,6 +1255,10 @@ static int variableParams(const ZSTD_compressionParameters paramConstraints, var
         res[j] = tlen_ind;
         j++;
     }
+    if(!paramConstraints.strategy) {
+        res[j] = strt_ind;
+        j++;
+    }
     return j;
 }
 
@@ -1285,6 +1293,7 @@ static void paramVaryOnce(const varInds_t paramIndex, const int amt, ZSTD_compre
         case tlen_ind: 
             ptr->targetLength = tlen_table[MAX(0, MIN(TLEN_RANGE - 1, tlen_inv(ptr->targetLength) + amt))];
             break;
+        case strt_ind: ptr->strategy     += amt; break;
         default: break;
     }
 }
@@ -1301,37 +1310,19 @@ static void paramVariation(ZSTD_compressionParameters* ptr, const varInds_t* var
             const U32 changeID = FUZ_rand(&g_rand) % (varyLen << 1);
             paramVaryOnce(varyParams[changeID >> 1], ((changeID & 1) << 1) - 1, &p);
         }
-        validated = !ZSTD_isError(ZSTD_checkCParams(p));
+        validated = !ZSTD_isError(ZSTD_checkCParams(p)) && p.strategy > 0;
     }
     *ptr = p;
 }
 
-/* maybe put strategy back in */
-static void paramVariationWithStrategy(ZSTD_compressionParameters* ptr, const varInds_t* varyParams, const int varyLen, const U32 nbChanges)
-{
-    ZSTD_compressionParameters p;
-    U32 validated = 0;
-    while (!validated) {
-        U32 i;
-        p = *ptr;
-        for (i = 0 ; i < nbChanges ; i++) {
-            const U32 changeID = FUZ_rand(&g_rand) % ((varyLen + 1) << 1);
-            if(changeID < (U32)(varyLen << 1)) {
-                paramVaryOnce(varyParams[changeID >> 1], ((changeID & 1) << 1) - 1, &p);
-            } else {
-                p.strategy += ((FUZ_rand(&g_rand) % 2) << 1) - 1; /* +/- 1 */
-            }
-        }
-        validated = !ZSTD_isError(ZSTD_checkCParams(p));
-    }
-    *ptr = p;
-}
 /* length of memo table given free variables */
 static size_t memoTableLen(const varInds_t* varyParams, const int varyLen) {
     size_t arrayLen = 1;
     int i;
     for(i = 0; i < varyLen; i++) {
-        arrayLen *= rangetable[varyParams[i]];
+        if(varyParams[i] != strt_ind) {
+            arrayLen *= rangetable[varyParams[i]];
+        }
     }
     return arrayLen;
 }
@@ -1354,6 +1345,7 @@ static unsigned memoTableInd(const ZSTD_compressionParameters* ptr, const varInd
                 - ZSTD_SEARCHLENGTH_MIN; break;
             case tlen_ind: ind *= TLEN_RANGE; ind += tlen_inv(ptr->targetLength) 
                 - ZSTD_TARGETLENGTH_MIN; break;
+            case strt_ind: break;
         }
     }
     return ind;
@@ -1376,6 +1368,7 @@ static void memoTableIndInv(ZSTD_compressionParameters* ptr, const varInds_t* va
                 ind /= SLEN_RANGE; break;
             case tlen_ind: ptr->targetLength = tlen_table[(ind % TLEN_RANGE)];
                 ind /= TLEN_RANGE; break;
+            case strt_ind: break;
         }
     }
 }
@@ -1505,7 +1498,7 @@ static void playAround(FILE* f, winnerInfo_t* winners,
 {
     int nbVariations = 0;
     UTIL_time_t const clockStart = UTIL_getTime();
-    const U32 unconstrained[NUM_PARAMS] = { 0, 1, 2, 3, 4, 5 };
+    const U32 unconstrained[NUM_PARAMS] = { 0, 1, 2, 3, 4, 5, 6 };
 
 
     while (UTIL_clockSpanMicro(clockStart) < g_maxVariationTime) {
@@ -1513,7 +1506,7 @@ static void playAround(FILE* f, winnerInfo_t* winners,
         BYTE* b;
 
         if (nbVariations++ > g_maxNbVariations) break;
-        paramVariationWithStrategy(&p, unconstrained, NUM_PARAMS, 4);
+        paramVariation(&p, unconstrained, NUM_PARAMS, 4);
 
         /* exclude faster if already played params */
         if (FUZ_rand(&g_rand) & ((1 << *NB_TESTS_PLAYED(p))-1))
@@ -1715,8 +1708,7 @@ static int benchOnce(buffers_t buf, contexts_t ctx) {
         return 1;
     }
 
-    DISPLAY("Compression Ratio: %.3f  Compress Speed: %.1f MB/s Decompress Speed: %.1f MB/s\n", (double)buf.srcSize / testResult.cSize, 
-        (double)testResult.cSpeed / (1 MB), (double)testResult.dSpeed / (1 MB));
+    BMK_printWinner(stdout, CUSTOM_LEVEL, testResult, g_params, buf.srcSize);
     return 0;
 }
 
@@ -1766,7 +1758,7 @@ int benchFiles(const char** fileNamesTable, int nbFiles, const char* dictFileNam
 }
 
 /* Benchmarking which stops when we are sufficiently sure the solution is infeasible / worse than the winner */
-#define VARIANCE 1.1 
+#define VARIANCE 1.2 
 static int allBench(BMK_result_t* resultPtr,
                 const buffers_t buf, const contexts_t ctx,
                 const ZSTD_compressionParameters cParams,
@@ -1918,9 +1910,9 @@ static int benchMemo(BMK_result_t* resultPtr,
  * all generation after random should be sanitized. (maybe sanitize random)
  */
 static winnerInfo_t climbOnce(const constraint_t target, 
-                const varInds_t* varArray, const int varLen, 
-                U8* const memoTable,
-                const buffers_t buf, const contexts_t ctx,
+                const varInds_t* varArray, const int varLen, ZSTD_strategy strat,
+                U8** memoTableArray, 
+                buffers_t buf, contexts_t ctx,
                 const ZSTD_compressionParameters init) {
     /* 
      * cparam - currently considered 'center'
@@ -1931,6 +1923,8 @@ static winnerInfo_t climbOnce(const constraint_t target,
     winnerInfo_t candidateInfo, winnerInfo;
     int better = 1;
     int feas = 0;
+    varInds_t varNew[NUM_PARAMS];
+    int varLenNew = sanitizeVarArray(varNew, varLen, varArray, strat);
 
     winnerInfo = initWinnerInfo(init);
     candidateInfo = winnerInfo;
@@ -1951,15 +1945,20 @@ static winnerInfo_t climbOnce(const constraint_t target,
                 for(offset = -1; offset <= 1; offset += 2) {
                     candidateInfo.params = cparam;
                     paramVaryOnce(varArray[i], offset, &candidateInfo.params); 
-                    candidateInfo.params = sanitizeParams(candidateInfo.params);
-                    if(!ZSTD_isError(ZSTD_checkCParams(candidateInfo.params))) {
-                        int res = benchMemo(&candidateInfo.result,
+                    
+                    if(!ZSTD_isError(ZSTD_checkCParams(candidateInfo.params)) && candidateInfo.params.strategy > 0) {
+                        int res;
+                        if(strat != candidateInfo.params.strategy) { /* maybe only try strategy switching after exhausting non-switching solutions? */
+                            strat = candidateInfo.params.strategy;
+                            varLenNew = sanitizeVarArray(varNew, varLen, varArray, strat);
+                        }
+                        res = benchMemo(&candidateInfo.result,
                             buf, ctx,
-                            candidateInfo.params, target, &winnerInfo.result, memoTable,
-                            varArray, varLen, feas);
+                            sanitizeParams(candidateInfo.params), target, &winnerInfo.result, memoTableArray[strat],
+                            varNew, varLenNew, feas);
                         if(res == BETTER_RESULT) { /* synonymous with better when called w/ infeasibleBM */
                             winnerInfo = candidateInfo;
-                            BMK_printWinnerOpt(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, target, buf.srcSize);
+                            BMK_printWinnerOpt(stdout, CUSTOM_LEVEL, winnerInfo.result, sanitizeParams(winnerInfo.params), target, buf.srcSize);
                             better = 1;
                             if(compareResultLT(bestFeasible1.result, winnerInfo.result, target, buf.srcSize)) {
                                 bestFeasible1 = winnerInfo;
@@ -1974,22 +1973,29 @@ static winnerInfo_t climbOnce(const constraint_t target,
             }
 
             for(dist = 2; dist < varLen + 2; dist++) { /* varLen is # dimensions */
-                for(i = 0; i < 2 * varLen + 2; i++) {
+                for(i = 0; i < (1 << varLen) / varLen + 2; i++) {
                     int res;
                     candidateInfo.params = cparam;
                     /* param error checking already done here */
                     paramVariation(&candidateInfo.params, varArray, varLen, dist);
+
+                    if(strat != candidateInfo.params.strategy) {
+                        strat = candidateInfo.params.strategy;
+                        varLenNew = sanitizeVarArray(varNew, varLen, varArray, strat);
+                    }
+
                     res = benchMemo(&candidateInfo.result,
                         buf, ctx, 
-                        candidateInfo.params, target, &winnerInfo.result, memoTable,
-                        varArray, varLen, feas);
+                        sanitizeParams(candidateInfo.params), target, &winnerInfo.result, memoTableArray[strat],
+                        varNew, varLenNew, feas);
                     if(res == BETTER_RESULT) { /* synonymous with better in this case*/
                         winnerInfo = candidateInfo;
-                        BMK_printWinnerOpt(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, target, buf.srcSize);
+                        BMK_printWinnerOpt(stdout, CUSTOM_LEVEL, winnerInfo.result, sanitizeParams(winnerInfo.params), target, buf.srcSize);
                         better = 1;
                         if(compareResultLT(bestFeasible1.result, winnerInfo.result, target, buf.srcSize)) {
                             bestFeasible1 = winnerInfo;
                         }
+                        break;
                     }
 
                 }
@@ -2026,10 +2032,11 @@ static winnerInfo_t optimizeFixedStrategy(
     const constraint_t target, ZSTD_compressionParameters paramTarget,
     const ZSTD_strategy strat, 
     const varInds_t* varArray, const int varLen,
-    U8* const memoTable, const int tries) {
+    U8** memoTableArray, const int tries) {
     int i = 0;
     varInds_t varNew[NUM_PARAMS];
     int varLenNew = sanitizeVarArray(varNew, varLen, varArray, strat);
+
     ZSTD_compressionParameters init;
     winnerInfo_t winnerInfo, candidateInfo; 
     winnerInfo = initWinnerInfo(emptyParams());
@@ -2043,8 +2050,8 @@ static winnerInfo_t optimizeFixedStrategy(
 
     while(i < tries) {
         DEBUGOUTPUT("Restart\n"); 
-        randomConstrainedParams(&init, varNew, varLenNew, memoTable);
-        candidateInfo = climbOnce(target, varNew, varLenNew, memoTable, buf, ctx, init);
+        randomConstrainedParams(&init, varNew, varLenNew, memoTableArray[strat]);
+        candidateInfo = climbOnce(target, varArray, varLen, strat, memoTableArray, buf, ctx, init);
         if(compareResultLT(winnerInfo.result, candidateInfo.result, target, buf.srcSize)) {
             winnerInfo = candidateInfo;
             BMK_printWinnerOpt(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, target, buf.srcSize);
@@ -2226,7 +2233,6 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_
     findClockGranularity();
 
     {   
-        varInds_t varNew[NUM_PARAMS];
         ZSTD_compressionParameters CParams;
 
         /* find best solution from default params */
@@ -2266,8 +2272,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_
 
                 { 
                     /* one iterations of hill climbing with the level-defined parameters. */
-                    int varLenNew = sanitizeVarArray(varNew, varLen, varArray, st);
-                    winnerInfo_t w1 = climbOnce(target, varNew, varLenNew, allMT[st], 
+                    winnerInfo_t w1 = climbOnce(target, varArray, varLen, st, allMT, 
                         buf, ctx, winner.params);
                     if(compareResultLT(winner.result, w1.result, target, buf.srcSize)) {
                         winner = w1;
@@ -2279,7 +2284,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_
                     DEBUGOUTPUT("StrategySwitch: %s\n", g_stratName[st]);
                     
                     wc = optimizeFixedStrategy(buf, ctx, target, paramTarget, 
-                        st, varArray, varLen, allMT[st], tries);
+                        st, varArray, varLen, allMT, tries);
 
                     if(compareResultLT(winner.result, wc.result, target, buf.srcSize)) {
                         winner = wc;
@@ -2292,7 +2297,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_
                 }
             } else {
                 winner = optimizeFixedStrategy(buf, ctx, target, paramTarget, paramTarget.strategy, 
-                    varArray, varLen, allMT[paramTarget.strategy], g_maxTries);
+                    varArray, varLen, allMT, g_maxTries);
             }
 
         }
@@ -2387,14 +2392,13 @@ static int badusage(const char* exename)
 #define PARSE_SUB_ARGS(stringLong, stringShort, variable) { if (longCommandWArg(&argument, stringLong) || longCommandWArg(&argument, stringShort)) { variable = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; } }
 #define PARSE_CPARAMS(variable)                                            \
 {                                                                          \
-    PARSE_SUB_ARGS("windowLog=",       "wlog=",  variable.vals[wlog_ind]); \
-    PARSE_SUB_ARGS("chainLog=" ,       "clog=",  variable.vals[clog_ind]); \
-    PARSE_SUB_ARGS("hashLog=",         "hlog=",  variable.vals[hlog_ind]); \
-    PARSE_SUB_ARGS("searchLog=" ,      "slog=",  variable.vals[slog_ind]); \
-    PARSE_SUB_ARGS("searchLength=",    "slen=",  variable.vals[slen_ind]); \
-    PARSE_SUB_ARGS("targetLength=" ,   "tlen=",  variable.vals[tlen_ind]); \
-    PARSE_SUB_ARGS("strategy=",        "strat=", variable.vals[strt_ind]); \
-    PARSE_SUB_ARGS("forceAttachDict=", "fad="  , variable.vals[strt_ind]); \
+    PARSE_SUB_ARGS("windowLog=",       "wlog=",  variable.windowLog);      \
+    PARSE_SUB_ARGS("chainLog=" ,       "clog=",  variable.chainLog);       \
+    PARSE_SUB_ARGS("hashLog=",         "hlog=",  variable.hashLog);        \
+    PARSE_SUB_ARGS("searchLog=" ,      "slog=",  variable.searchLog);      \
+    PARSE_SUB_ARGS("searchLength=",    "slen=",  variable.searchLength);   \
+    PARSE_SUB_ARGS("targetLength=" ,   "tlen=",  variable.targetLength);   \
+    PARSE_SUB_ARGS("strategy=",        "strat=", variable.strategy);       \
 }
 
 int main(int argc, const char** argv)

From a6df961497b766546db31b7876aef5b3718c17a4 Mon Sep 17 00:00:00 2001
From: Eden Zik 
Date: Mon, 13 Aug 2018 20:28:52 -0400
Subject: [PATCH 155/372] Cmake now builds with CMAKE_BUILD_TYPE=Release by
 default, both while being invoked from the main Makefile (via cmakebuild) or
 directly from the build/cmake directory. Suggested by @pdknsk (#1081).

---
 Makefile                   |  2 +-
 README.md                  |  2 ++
 build/.gitignore           | 11 +++++++++++
 build/cmake/CMakeLists.txt |  4 ++++
 4 files changed, 18 insertions(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index f67791275..6c4ab9c9b 100644
--- a/Makefile
+++ b/Makefile
@@ -114,7 +114,7 @@ clean:
 ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD DragonFly NetBSD MSYS_NT))
 
 HOST_OS = POSIX
-CMAKE_PARAMS = -DZSTD_BUILD_CONTRIB:BOOL=ON -DZSTD_BUILD_STATIC:BOOL=ON -DZSTD_BUILD_TESTS:BOOL=ON -DZSTD_ZLIB_SUPPORT:BOOL=ON -DZSTD_LZMA_SUPPORT:BOOL=ON
+CMAKE_PARAMS = -DZSTD_BUILD_CONTRIB:BOOL=ON -DZSTD_BUILD_STATIC:BOOL=ON -DZSTD_BUILD_TESTS:BOOL=ON -DZSTD_ZLIB_SUPPORT:BOOL=ON -DZSTD_LZMA_SUPPORT:BOOL=ON -DCMAKE_BUILD_TYPE=Release 
 
 .PHONY: list
 list:
diff --git a/README.md b/README.md
index 17edecb71..dc99dc0fd 100644
--- a/README.md
+++ b/README.md
@@ -121,6 +121,8 @@ A `cmake` project generator is provided within `build/cmake`.
 It can generate Makefiles or other build scripts
 to create `zstd` binary, and `libzstd` dynamic and static libraries.
 
+By default, `CMAKE_BUILD_TYPE` is set to `Release`.
+
 #### Meson
 
 A Meson project is provided within `contrib/meson`.
diff --git a/build/.gitignore b/build/.gitignore
index b00e709ba..1ceb70ebc 100644
--- a/build/.gitignore
+++ b/build/.gitignore
@@ -18,3 +18,14 @@ Studio*
 
 # CMake
 cmake/build/
+CMakeCache.txt
+CMakeFiles
+CMakeScripts
+Testing
+Makefile
+cmake_install.cmake
+install_manifest.txt
+compile_commands.json
+CTestTestfile.cmake
+build
+lib
diff --git a/build/cmake/CMakeLists.txt b/build/cmake/CMakeLists.txt
index fd9bc2b1e..1e2921de8 100644
--- a/build/cmake/CMakeLists.txt
+++ b/build/cmake/CMakeLists.txt
@@ -10,6 +10,10 @@
 PROJECT(zstd)
 CMAKE_MINIMUM_REQUIRED(VERSION 2.8.9)
 SET(ZSTD_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../..")
+
+# Ensure Release build even if not invoked via Makefile
+SET(CMAKE_BUILD_TYPE "Release")
+
 LIST(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/CMakeModules")
 INCLUDE(GNUInstallDirs)
 

From 614aaa3ae152aa13c3b17ae01a39f151be6c1c56 Mon Sep 17 00:00:00 2001
From: George Lu 
Date: Mon, 13 Aug 2018 16:38:51 -0700
Subject: [PATCH 156/372] rebase clevel

---
 tests/paramgrill.c | 63 ++++++++++++++++++++++++----------------------
 1 file changed, 33 insertions(+), 30 deletions(-)

diff --git a/tests/paramgrill.c b/tests/paramgrill.c
index 955032e84..58f2e13bc 100644
--- a/tests/paramgrill.c
+++ b/tests/paramgrill.c
@@ -65,6 +65,7 @@ static const int g_maxNbVariations = 64;
 #define MIN(a,b)   ( (a) < (b) ? (a) : (b) )
 #define MAX(a,b)   ( (a) > (b) ? (a) : (b) )
 #define CUSTOM_LEVEL 99
+#define BASE_CLEVEL 1
 
 /* indices for each of the variables */
 typedef enum {
@@ -112,7 +113,7 @@ static U32 g_singleRun = 0;
 static U32 g_optimizer = 0;
 static U32 g_target = 0;
 static U32 g_noSeed = 0;
-static ZSTD_compressionParameters g_params;
+static ZSTD_compressionParameters g_params; /* Initialized at the beginning of main w/ emptyParams() function */
 static UTIL_time_t g_time; /* to be used to compare solution finding speeds to compare to original */
 
 
@@ -523,6 +524,7 @@ static void freeBuffers(const buffers_t b) {
         free(b.resPtrs[0]);
     }
     free(b.resPtrs);
+    free(b.resSizes);
 }
 
 /* allocates buffer's arguments. returns success / failuere */
@@ -532,7 +534,7 @@ static int createBuffers(buffers_t* buff, const char* const * const fileNamesTab
     size_t pos = 0;
     size_t n;
     U64 const totalSizeToLoad = UTIL_getTotalFileSize(fileNamesTable, (U32)nbFiles);
-    size_t benchedSize = MIN(BMK_findMaxMem(totalSizeToLoad * 3) / 3, totalSizeToLoad);
+    const size_t benchedSize = MIN(BMK_findMaxMem(totalSizeToLoad * 3) / 3, totalSizeToLoad);
     const size_t blockSize = g_blockSize ? g_blockSize : totalSizeToLoad;
     U32 const maxNbBlocks = (U32) ((totalSizeToLoad + (blockSize-1)) / blockSize) + (U32)nbFiles;
     U32 blockNb = 0;
@@ -671,6 +673,7 @@ static int createContexts(contexts_t* ctx, const char* dictFileName) {
         freeContexts(*ctx);
         return 1;
     }
+    fclose(f);
     return 0;
 }
 
@@ -824,7 +827,7 @@ static BMK_return_t BMK_benchMemInvertible(const buffers_t buf, const contexts_t
 static int BMK_benchParam(BMK_result_t* resultPtr,
                 buffers_t buf, contexts_t ctx,
                 const ZSTD_compressionParameters cParams) {
-    BMK_return_t res = BMK_benchMemInvertible(buf, ctx, 0, &cParams, BMK_both, BMK_timeMode, 3);
+    BMK_return_t res = BMK_benchMemInvertible(buf, ctx, BASE_CLEVEL, &cParams, BMK_both, BMK_timeMode, 3);
     *resultPtr = res.result;
     return res.error;
 }
@@ -840,16 +843,16 @@ static int BMK_benchParam(BMK_result_t* resultPtr,
 #define SIZE_RESULT 5
 /* maybe have epsilon-eq to limit table size? */
 static int speedSizeCompare(BMK_result_t r1, BMK_result_t r2) {
-    if(r1.cSpeed > r2.cSpeed) {
-        if(r1.cSize <= r2.cSize) {
-            return WORSE_RESULT;
-        }
-        return SIZE_RESULT; /* r2 is smaller but not faster. */
-    } else {
+    if(r1.cSpeed < r2.cSpeed) {
         if(r1.cSize >= r2.cSize) {
             return BETTER_RESULT;
         }
-        return SPEED_RESULT; /* r2 is faster but not smaller */
+        return SPEED_RESULT; /* r2 is smaller but not faster. */
+    } else {
+        if(r1.cSize <= r2.cSize) {
+            return WORSE_RESULT;
+        }
+        return SIZE_RESULT; /* r2 is faster but not smaller */
     }
 }
 
@@ -875,12 +878,12 @@ static int insertWinner(winnerInfo_t w, constraint_t targetConstraints) {
     }
 
     while(cur_node->next != NULL) {
-        switch(speedSizeCompare(r, cur_node->res.result)) {
-            case BETTER_RESULT:
+        switch(speedSizeCompare(cur_node->res.result, r)) {
+            case WORSE_RESULT:
             {
                 return 1; /* never insert if better */
             }
-            case WORSE_RESULT:
+            case BETTER_RESULT:
             {
                 winner_ll_node* tmp;
                 cur_node->res = cur_node->next->res;
@@ -889,12 +892,12 @@ static int insertWinner(winnerInfo_t w, constraint_t targetConstraints) {
                 free(tmp);
                 break; 
             }
-            case SPEED_RESULT:
+            case SIZE_RESULT:
             {
                 cur_node = cur_node->next;
                 break;
             }
-            case SIZE_RESULT: /* insert after first size result, then return */
+            case SPEED_RESULT: /* insert after first size result, then return */
             {
                 winner_ll_node* newnode = malloc(sizeof(winner_ll_node));
                 if(newnode == NULL) {
@@ -911,17 +914,17 @@ static int insertWinner(winnerInfo_t w, constraint_t targetConstraints) {
     }
 
     assert(cur_node->next == NULL);
-    switch(speedSizeCompare(r, cur_node->res.result)) {
-        case BETTER_RESULT:
+    switch(speedSizeCompare(cur_node->res.result, r)) {
+        case WORSE_RESULT:
         {
             return 1; /* never insert if better */
         }
-        case WORSE_RESULT:
+        case BETTER_RESULT:
         {
             cur_node->res = w;
             return 0;
         }
-        case SPEED_RESULT:
+        case SIZE_RESULT:
         {
             winner_ll_node* newnode = malloc(sizeof(winner_ll_node));
             if(newnode == NULL) {
@@ -932,7 +935,7 @@ static int insertWinner(winnerInfo_t w, constraint_t targetConstraints) {
             cur_node->next = newnode;
             return 0;
         }
-        case SIZE_RESULT: /* insert before first size result, then return */
+        case SPEED_RESULT: /* insert before first size result, then return */
         {
             winner_ll_node* newnode = malloc(sizeof(winner_ll_node));
             if(newnode == NULL) {
@@ -1448,7 +1451,7 @@ static void freeMemoTableArray(U8** mtAll) {
 /* takes unsanitized varyParams */
 static U8** createMemoTableArray(ZSTD_compressionParameters paramConstraints, constraint_t target, const varInds_t* varyParams, const int varyLen, const size_t srcSize) {
     varInds_t varNew[NUM_PARAMS];
-    U8** mtAll = calloc(sizeof(U8*),(ZSTD_btultra + 1));
+    U8** mtAll = (U8**)calloc(sizeof(U8*),(ZSTD_btultra + 1));
     int i;
     if(mtAll == NULL) {
         return NULL;
@@ -1467,7 +1470,7 @@ static U8** createMemoTableArray(ZSTD_compressionParameters paramConstraints, co
     return mtAll;
 }
 
-static ZSTD_compressionParameters maskParams(ZSTD_compressionParameters base, ZSTD_compressionParameters mask) {
+static ZSTD_compressionParameters overwriteParams(ZSTD_compressionParameters base, ZSTD_compressionParameters mask) {
     base.windowLog = mask.windowLog ? mask.windowLog : base.windowLog;
     base.chainLog = mask.chainLog ? mask.chainLog : base.chainLog;
     base.hashLog = mask.hashLog ? mask.hashLog : base.hashLog;
@@ -1744,7 +1747,7 @@ int benchFiles(const char** fileNamesTable, int nbFiles, const char* dictFileNam
         DISPLAY("using %d Files : \n", nbFiles);
     }
 
-    g_params = ZSTD_adjustCParams(maskParams(ZSTD_getCParams(cLevel, maxBlockSize, ctx.dictSize), g_params), maxBlockSize, ctx.dictSize);
+    g_params = ZSTD_adjustCParams(overwriteParams(ZSTD_getCParams(cLevel, maxBlockSize, ctx.dictSize), g_params), maxBlockSize, ctx.dictSize);
 
     if(g_singleRun) {
         ret = benchOnce(buf, ctx);
@@ -1771,7 +1774,7 @@ static int allBench(BMK_result_t* resultPtr,
     double winnerRS;
 
     /* initial benchmarking, gives exact ratio and memory, warms up future runs */
-    benchres = BMK_benchMemInvertible(buf, ctx, 0, &cParams, BMK_both, BMK_iterMode, 1);
+    benchres = BMK_benchMemInvertible(buf, ctx, BASE_CLEVEL, &cParams, BMK_both, BMK_iterMode, 1);
 
     winnerRS = resultScore(*winnerResult, buf.srcSize, target);
     DEBUGOUTPUT("WinnerScore: %f\n ", winnerRS);
@@ -1806,14 +1809,14 @@ static int allBench(BMK_result_t* resultPtr,
 
     /* second run, if first run is too short, gives approximate cSpeed + dSpeed */
     if(loopDurationC < TIMELOOP_NANOSEC / 10) {
-        BMK_return_t benchres2 = BMK_benchMemInvertible(buf, ctx, 0, &cParams, BMK_compressOnly, BMK_iterMode, 1);
+        BMK_return_t benchres2 = BMK_benchMemInvertible(buf, ctx, BASE_CLEVEL, &cParams, BMK_compressOnly, BMK_iterMode, 1);
         if(benchres2.error) {
             return ERROR_RESULT;
         }
         benchres = benchres2;
     }
     if(loopDurationD < TIMELOOP_NANOSEC / 10) {
-        BMK_return_t benchres2 = BMK_benchMemInvertible(buf, ctx, 0, &cParams, BMK_decodeOnly, BMK_iterMode, 1);
+        BMK_return_t benchres2 = BMK_benchMemInvertible(buf, ctx, BASE_CLEVEL, &cParams, BMK_decodeOnly, BMK_iterMode, 1);
         if(benchres2.error) {
             return ERROR_RESULT;
         }
@@ -1836,7 +1839,7 @@ static int allBench(BMK_result_t* resultPtr,
 
     /* Final full run if estimates are unclear */
     if(loopDurationC < TIMELOOP_NANOSEC) {
-        BMK_return_t benchres2 = BMK_benchMemInvertible(buf, ctx, 0, &cParams, BMK_compressOnly, BMK_timeMode, 1);
+        BMK_return_t benchres2 = BMK_benchMemInvertible(buf, ctx, BASE_CLEVEL, &cParams, BMK_compressOnly, BMK_timeMode, 1);
         if(benchres2.error) {
             return ERROR_RESULT;
         }
@@ -1844,7 +1847,7 @@ static int allBench(BMK_result_t* resultPtr,
     } 
 
     if(loopDurationD < TIMELOOP_NANOSEC) {
-        BMK_return_t benchres2 = BMK_benchMemInvertible(buf, ctx, 0, &cParams, BMK_decodeOnly, BMK_timeMode, 1);
+        BMK_return_t benchres2 = BMK_benchMemInvertible(buf, ctx, BASE_CLEVEL, &cParams, BMK_decodeOnly, BMK_timeMode, 1);
         if(benchres2.error) {
             return ERROR_RESULT;
         }
@@ -2245,7 +2248,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_
                 int i;
                 for (i=1; i<=maxSeeds; i++) {
                     int ec;
-                    CParams = maskParams(ZSTD_getCParams(i, maxBlockSize, ctx.dictSize), paramTarget);
+                    CParams = overwriteParams(ZSTD_getCParams(i, maxBlockSize, ctx.dictSize), paramTarget);
                     ec = BMK_benchParam(&candidate, buf, ctx, CParams);
                     BMK_printWinnerOpt(stdout, i, candidate, CParams, target, buf.srcSize);
 
@@ -2439,11 +2442,11 @@ int main(int argc, const char** argv)
                 PARSE_SUB_ARGS("compressionSpeed=" ,  "cSpeed=", target.cSpeed);
                 PARSE_SUB_ARGS("decompressionSpeed=", "dSpeed=", target.dSpeed); 
                 PARSE_SUB_ARGS("compressionMemory=" , "cMem=", target.cMem);
-                PARSE_SUB_ARGS("level=", "lvl=", cLevel);
                 PARSE_SUB_ARGS("strict=", "stc=", g_strictness);
                 PARSE_SUB_ARGS("preferSpeed=", "prfSpd=", g_speedMultiplier);
                 PARSE_SUB_ARGS("preferRatio=", "prfRto=", g_ratioMultiplier);
                 PARSE_SUB_ARGS("maxTries=", "tries=", g_maxTries);
+                if (longCommandWArg(&argument, "level=") || longCommandWArg(&argument, "lvl=")) { cLevel = readU32FromChar(&argument); g_optmode = 1; if (argument[0]==',') { argument++; continue; } else break; }
 
                 DISPLAY("invalid optimization parameter \n");
                 return 1;

From 3e4617ef54a423d9c839462b79595162466f05e5 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Tue, 14 Aug 2018 11:49:25 -0700
Subject: [PATCH 157/372] frameProgression reports nbActiveWorkers and output
 flushed

---
 lib/compress/zstd_compress.c   |  2 ++
 lib/compress/zstdmt_compress.c | 18 +++++++++++-------
 lib/zstd.h                     | 10 ++++++----
 programs/fileio.c              | 17 +++++++----------
 4 files changed, 26 insertions(+), 21 deletions(-)

diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index 1412c1d6a..4bc18d2ec 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -900,7 +900,9 @@ ZSTD_frameProgression ZSTD_getFrameProgression(const ZSTD_CCtx* cctx)
         fp.ingested = cctx->consumedSrcSize + buffered;
         fp.consumed = cctx->consumedSrcSize;
         fp.produced = cctx->producedCSize;
+        fp.flushed  = cctx->producedCSize;   /* simplified; some data might still be left within streaming output buffer */
         fp.currentJobID = 0;
+        fp.nbActiveWorkers = 0;
         return fp;
 }   }
 
diff --git a/lib/compress/zstdmt_compress.c b/lib/compress/zstdmt_compress.c
index 49502bd0d..b61cd50ee 100644
--- a/lib/compress/zstdmt_compress.c
+++ b/lib/compress/zstdmt_compress.c
@@ -1058,7 +1058,7 @@ static size_t ZSTDMT_resize(ZSTDMT_CCtx* mtctx, unsigned nbWorkers)
 
 
 /*! ZSTDMT_updateCParams_whileCompressing() :
- *  Updates only a selected set of compression parameters, to remain compatible with current frame.
+ *  Updates a selected set of compression parameters, remaining compatible with currently active frame.
  *  New parameters will be applied to next compression job. */
 void ZSTDMT_updateCParams_whileCompressing(ZSTDMT_CCtx* mtctx, const ZSTD_CCtx_params* cctxParams)
 {
@@ -1076,27 +1076,31 @@ void ZSTDMT_updateCParams_whileCompressing(ZSTDMT_CCtx* mtctx, const ZSTD_CCtx_p
 /* ZSTDMT_getFrameProgression():
  * tells how much data has been consumed (input) and produced (output) for current frame.
  * able to count progression inside worker threads.
- * Note : mutex will be acquired during statistics collection. */
+ * Note : mutex will be acquired during statistics collection inside workers. */
 ZSTD_frameProgression ZSTDMT_getFrameProgression(ZSTDMT_CCtx* mtctx)
 {
     ZSTD_frameProgression fps;
     DEBUGLOG(5, "ZSTDMT_getFrameProgression");
     fps.ingested = mtctx->consumed + mtctx->inBuff.filled;
     fps.consumed = mtctx->consumed;
-    fps.produced = mtctx->produced;
+    fps.produced = fps.flushed = mtctx->produced;
     fps.currentJobID = mtctx->nextJobID;
+    fps.nbActiveWorkers = 0;
     {   unsigned jobNb;
         unsigned lastJobNb = mtctx->nextJobID + mtctx->jobReady; assert(mtctx->jobReady <= 1);
         DEBUGLOG(6, "ZSTDMT_getFrameProgression: jobs: from %u to <%u (jobReady:%u)",
                     mtctx->doneJobID, lastJobNb, mtctx->jobReady)
         for (jobNb = mtctx->doneJobID ; jobNb < lastJobNb ; jobNb++) {
             unsigned const wJobID = jobNb & mtctx->jobIDMask;
-            ZSTD_pthread_mutex_lock(&mtctx->jobs[wJobID].job_mutex);
-            {   size_t const cResult = mtctx->jobs[wJobID].cSize;
+            ZSTDMT_jobDescription* jobPtr = &mtctx->jobs[wJobID];
+            ZSTD_pthread_mutex_lock(&jobPtr->job_mutex);
+            {   size_t const cResult = jobPtr->cSize;
                 size_t const produced = ZSTD_isError(cResult) ? 0 : cResult;
-                fps.ingested += mtctx->jobs[wJobID].src.size;
-                fps.consumed += mtctx->jobs[wJobID].consumed;
+                fps.ingested += jobPtr->src.size;
+                fps.consumed += jobPtr->consumed;
                 fps.produced += produced;
+                fps.flushed  += jobPtr->dstFlushed;
+                fps.nbActiveWorkers += (jobPtr->consumed < jobPtr->src.size);
             }
             ZSTD_pthread_mutex_unlock(&mtctx->jobs[wJobID].job_mutex);
         }
diff --git a/lib/zstd.h b/lib/zstd.h
index edd0079c9..02e447b30 100644
--- a/lib/zstd.h
+++ b/lib/zstd.h
@@ -732,10 +732,12 @@ ZSTDLIB_API size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledg
 
 
 typedef struct {
-    unsigned long long ingested;
-    unsigned long long consumed;
-    unsigned long long produced;
-    unsigned currentJobID;
+    unsigned long long ingested;   /* nb input bytes read and buffered */
+    unsigned long long consumed;   /* nb input bytes actually compressed */
+    unsigned long long produced;   /* nb of compressed bytes generated and buffered */
+    unsigned long long flushed;    /* nb of compressed bytes flushed : not provided; can be tracked from caller side */
+    unsigned currentJobID;         /* MT only : latest started job nb */
+    unsigned nbActiveWorkers;      /* MT only : nb of workers actively compressing at probe time */
 } ZSTD_frameProgression;
 
 /* ZSTD_getFrameProgression():
diff --git a/programs/fileio.c b/programs/fileio.c
index b29936191..68b2f1593 100644
--- a/programs/fileio.c
+++ b/programs/fileio.c
@@ -800,16 +800,17 @@ FIO_compressZstdFrame(const cRess_t* ressPtr,
 
                 /* check output speed */
                 if (zfp.currentJobID > 1) {
-                    static ZSTD_frameProgression cpszfp = { 0, 0, 0, 0 };
-                    static unsigned long long lastFlushedSize = 0;
+                    static ZSTD_frameProgression cpszfp = { 0, 0, 0, 0, 0, 0 };
 
                     unsigned long long newlyProduced = zfp.produced - cpszfp.produced;
-                    unsigned long long newlyFlushed = compressedfilesize - lastFlushedSize;
+                    unsigned long long newlyFlushed = zfp.flushed - cpszfp.flushed;
                     assert(zfp.produced >= cpszfp.produced);
 
+                    cpszfp = zfp;
+
                     if ( (zfp.ingested == cpszfp.ingested)
                       && (zfp.consumed == cpszfp.consumed) ) {
-                        DISPLAYLEVEL(6, "no data read nor consumed : buffers are full (?) or compression is slow + input has reached its limit. If buffers full : output is too slow => slow down \n")
+                        DISPLAYLEVEL(2, "no data read nor consumed : buffers are full (?) output is too slow => slow down ; or compression is slow + input has reached its limit => can't tell \n")
                         speedChange = slower;
                     }
 
@@ -818,8 +819,6 @@ FIO_compressZstdFrame(const cRess_t* ressPtr,
                         DISPLAYLEVEL(6, "production faster than flushing (%llu > %llu) but there is still %u bytes to flush => slow down \n", newlyProduced, newlyFlushed, (U32)stillToFlush);
                         speedChange = slower;
                     }
-                    cpszfp = zfp;
-                    lastFlushedSize = compressedfilesize;
                 }
 
                 /* course correct only if there is at least one new job completed */
@@ -832,14 +831,12 @@ FIO_compressZstdFrame(const cRess_t* ressPtr,
                             DISPLAYLEVEL(6, "input is never blocked => input is too slow \n");
                             speedChange = slower;
                         } else if (speedChange == noChange) {
-                            static ZSTD_frameProgression csuzfp = { 0, 0, 0, 0 };
-                            static unsigned long long lastFlushedSize = 0;
+                            static ZSTD_frameProgression csuzfp = { 0, 0, 0, 0, 0, 0 };
                             unsigned long long newlyIngested = zfp.ingested - csuzfp.ingested;
                             unsigned long long newlyConsumed = zfp.consumed - csuzfp.consumed;
                             unsigned long long newlyProduced = zfp.produced - csuzfp.produced;
-                            unsigned long long newlyFlushed = compressedfilesize - lastFlushedSize;
+                            unsigned long long newlyFlushed = zfp.flushed - csuzfp.flushed;
                             csuzfp = zfp;
-                            lastFlushedSize = compressedfilesize;
                             assert(inputPresented > 0);
                             DISPLAYLEVEL(6, "input blocked %u/%u(%.2f) - ingested:%u vs %u:consumed - flushed:%u vs %u:produced \n",
                                             inputBlocked, inputPresented, (double)inputBlocked/inputPresented*100,

From 76acba025da5ead78532bfd0dff4de6af67a6506 Mon Sep 17 00:00:00 2001
From: George Lu 
Date: Tue, 14 Aug 2018 11:57:15 -0700
Subject: [PATCH 158/372] scan-build

---
 tests/paramgrill.c | 32 +++++++++++++++++++++++---------
 1 file changed, 23 insertions(+), 9 deletions(-)

diff --git a/tests/paramgrill.c b/tests/paramgrill.c
index 58f2e13bc..7e85bf832 100644
--- a/tests/paramgrill.c
+++ b/tests/paramgrill.c
@@ -176,14 +176,14 @@ static size_t BMK_findMaxMem(U64 requiredMem)
     requiredMem = (((requiredMem >> 26) + 1) << 26);
     if (requiredMem > maxMemory) requiredMem = maxMemory;
 
-    requiredMem += 2*step;
-    while (!testmem) {
-        requiredMem -= step;
+    requiredMem += 2 * step;
+    while (!testmem && requiredMem > 0) {
         testmem = malloc ((size_t)requiredMem);
+        requiredMem -= step;
     }
 
     free (testmem);
-    return (size_t) (requiredMem - step);
+    return (size_t) requiredMem;
 }
 
 
@@ -536,9 +536,14 @@ static int createBuffers(buffers_t* buff, const char* const * const fileNamesTab
     U64 const totalSizeToLoad = UTIL_getTotalFileSize(fileNamesTable, (U32)nbFiles);
     const size_t benchedSize = MIN(BMK_findMaxMem(totalSizeToLoad * 3) / 3, totalSizeToLoad);
     const size_t blockSize = g_blockSize ? g_blockSize : totalSizeToLoad;
-    U32 const maxNbBlocks = (U32) ((totalSizeToLoad + (blockSize-1)) / blockSize) + (U32)nbFiles;
+    U32 const maxNbBlocks = (U32) ((totalSizeToLoad + (blockSize-1)) / MAX(blockSize, 1)) + (U32)nbFiles;
     U32 blockNb = 0;
 
+    if(!totalSizeToLoad || !benchedSize) {
+        DISPLAY("Nothing to Bench\n");
+        return 1;
+    }
+    
     buff->srcPtrs = (const void**)calloc(maxNbBlocks, sizeof(void*));
     buff->srcSizes = (size_t*)malloc(maxNbBlocks * sizeof(size_t));
 
@@ -555,6 +560,7 @@ static int createBuffers(buffers_t* buff, const char* const * const fileNamesTab
         return 1;
     }
 
+
     buff->srcBuffer = malloc(benchedSize);
     buff->srcPtrs[0] = (const void*)buff->srcBuffer;
     buff->dstPtrs[0] = malloc(ZSTD_compressBound(benchedSize) + (maxNbBlocks * 1024));
@@ -613,6 +619,12 @@ static int createBuffers(buffers_t* buff, const char* const * const fileNamesTab
         fclose(f);
     }
 
+    if(!blockNb) {
+        DISPLAY("Failed to load any files\n");
+        freeBuffers(*buff);
+        return 1;
+    }
+
     buff->dstCapacities[0] = ZSTD_compressBound(buff->srcSizes[0]);
     buff->dstSizes[0] = buff->dstCapacities[0];
     buff->resSizes[0] = buff->srcSizes[0];
@@ -957,8 +969,6 @@ static int insertWinner(winnerInfo_t w, constraint_t targetConstraints) {
 static void BMK_printWinner(FILE* f, const U32 cLevel, const BMK_result_t result, const ZSTD_compressionParameters params, const size_t srcSize)
 {
     char lvlstr[15] = "Custom Level";
-    const U64 time = UTIL_clockSpanNano(g_time);
-    const U64 minutes = time / (60ULL * TIMELOOP_NANOSEC);
 
     fprintf(f, "\r%79s\r", "");
 
@@ -974,7 +984,11 @@ static void BMK_printWinner(FILE* f, const U32 cLevel, const BMK_result_t result
         "/* %s */   /* R:%5.3f at %5.1f MB/s - %5.1f MB/s */",
         lvlstr, (double)srcSize / result.cSize, (double)result.cSpeed / (1 MB), (double)result.dSpeed / (1 MB));
 
-    if(TIMED) { fprintf(f, " - %1lu:%2lu:%05.2f", (unsigned long) minutes / 60,(unsigned long) minutes % 60, (double)(time - minutes * TIMELOOP_NANOSEC * 60ULL)/TIMELOOP_NANOSEC); }
+    if(TIMED) {
+        const U64 time = UTIL_clockSpanNano(g_time);
+        const U64 minutes = time / (60ULL * TIMELOOP_NANOSEC);
+        fprintf(f, " - %1lu:%2lu:%05.2f", (unsigned long) minutes / 60,(unsigned long) minutes % 60, (double)(time - minutes * TIMELOOP_NANOSEC * 60ULL)/TIMELOOP_NANOSEC); 
+    }
     fprintf(f, "\n"); 
 }
 
@@ -2160,7 +2174,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_
     if(paramTarget.strategy) {
         varInds_t varNew[NUM_PARAMS];
         int varLenNew = sanitizeVarArray(varNew, varLen, varArray, paramTarget.strategy);
-        allMT = calloc(sizeof(U8), (ZSTD_btultra + 1));
+        allMT = (U8**)calloc(sizeof(U8*), (ZSTD_btultra + 1));
         if(allMT == NULL) {
             ret = 57;
             goto _cleanUp;

From f581ccd267244da3d0a4d8d6ba1cb8a1b191bfd8 Mon Sep 17 00:00:00 2001
From: George Lu 
Date: Tue, 31 Jul 2018 18:47:27 -0700
Subject: [PATCH 159/372] Doc Updates

Add option to pass in existing parameters in use
---
 tests/README.md    |  4 ++--
 tests/paramgrill.c | 13 +++++++++++++
 2 files changed, 15 insertions(+), 2 deletions(-)

diff --git a/tests/README.md b/tests/README.md
index 736916936..4d2f314d2 100644
--- a/tests/README.md
+++ b/tests/README.md
@@ -105,7 +105,8 @@ Full list of arguments
     t# - targetLength
     S# - strategy
     L# - level
- --zstd=      : Single run, parameter selection syntax same as zstdcli
+ --zstd=      : Single run, parameter selection syntax same as zstdcli. 
+                When invoked with --optimize, this represents the sample to exceed. 
  --optimize=  : find parameters to maximize compression ratio given parameters
                 Can use all --zstd= commands to constrain the type of solution found in addition to the following constraints
     cSpeed=   : Minimum compression speed
@@ -120,7 +121,6 @@ Full list of arguments
                 when determining overall winner (default 1 for both, higher = more valued).
     tries=    : Maximum number of random restarts on a single strategy before switching (Default 5)
                 Higher values will make optimizer run longer, more chances to find better solution.
- --optimize=  : same as -O with more verbose syntax 
  -P#          : generated sample compressibility 
  -t#          : Caps runtime of operation in seconds (default : 99999 seconds (about 27 hours )) 
  -v           : Prints Benchmarking output
diff --git a/tests/paramgrill.c b/tests/paramgrill.c
index 7e85bf832..47e0a9421 100644
--- a/tests/paramgrill.c
+++ b/tests/paramgrill.c
@@ -2234,6 +2234,19 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_
         BMK_printWinnerOpt(stdout, cLevel, winner.result, winner.params, target, buf.srcSize);
     }
 
+    if(g_singleRun) {
+        BMK_result_t res;
+        g_params = ZSTD_adjustCParams(maskParams(ZSTD_getCParams(cLevel, maxBlockSize, ctx.dictSize), g_params), maxBlockSize, ctx.dictSize);
+        if(BMK_benchParam(&res, buf, ctx, g_params)) {
+            ret = 45;
+            goto _cleanUp;
+        }
+        if(compareResultLT(winner.result, res, relaxTarget(target), buf.srcSize)) {
+            winner.result = res;
+            winner.params = g_params;
+        }
+    }
+
     /* bench */
     DISPLAY("\r%79s\r", "");
     if(nbFiles == 1) {

From 88dda922854550959bda7640f554b5f02e5d0d4f Mon Sep 17 00:00:00 2001
From: George Lu 
Date: Mon, 6 Aug 2018 15:08:35 -0700
Subject: [PATCH 160/372] Reduce Duplication

Change Defaults
Asserts actually disabled in paramgrill + fullbench
---
 tests/Makefile     |   4 +-
 tests/README.md    |   4 +-
 tests/paramgrill.c | 287 +++++++++++++++++++++------------------------
 3 files changed, 140 insertions(+), 155 deletions(-)

diff --git a/tests/Makefile b/tests/Makefile
index 81e685780..88a5d763c 100644
--- a/tests/Makefile
+++ b/tests/Makefile
@@ -129,7 +129,7 @@ zstdmt_d_%.o : $(ZSTDDIR)/decompress/%.c
 fullbench32: CPPFLAGS += -m32
 fullbench fullbench32 : CPPFLAGS += $(MULTITHREAD_CPP)
 fullbench fullbench32 : LDFLAGS += $(MULTITHREAD_LD)
-fullbench fullbench32 : DEBUGFLAGS =   # turn off assert() for speed measurements
+fullbench fullbench32 : DEBUGFLAGS = -DNDEBUG  # turn off assert() for speed measurements
 fullbench fullbench32 : $(ZSTD_FILES)
 fullbench fullbench32 : $(PRGDIR)/datagen.c $(PRGDIR)/bench.c fullbench.c
 	$(CC) $(FLAGS) $^ -o $@$(EXT)
@@ -200,7 +200,7 @@ zstreamtest-dll : $(ZSTDDIR)/common/xxhash.c  # xxh symbols not exposed from dll
 zstreamtest-dll : $(ZSTREAM_LOCAL_FILES)
 	$(CC) $(CPPFLAGS) $(CFLAGS) $(filter %.c,$^) $(LDFLAGS) -o $@$(EXT)
 
-paramgrill : DEBUGFLAGS =   # turn off assert() for speed measurements
+paramgrill : DEBUGFLAGS = -DNDEBUG  # turn off assert() for speed measurements
 paramgrill : $(ZSTD_FILES) $(PRGDIR)/bench.c $(PRGDIR)/datagen.c paramgrill.c
 	$(CC) $(FLAGS) $^ -lm -o $@$(EXT)
 
diff --git a/tests/README.md b/tests/README.md
index 4d2f314d2..3c35ecf01 100644
--- a/tests/README.md
+++ b/tests/README.md
@@ -118,8 +118,8 @@ Full list of arguments
                 (Lower value will begin with stronger strategies) (Default 90%)
     preferSpeed= / preferRatio=
               : Only affects lvl = invocations. Defines value placed on compression speed or ratio
-                when determining overall winner (default 1 for both, higher = more valued).
-    tries=    : Maximum number of random restarts on a single strategy before switching (Default 5)
+                when determining overall winner (default speed = 1, ratio = 5 for both, higher = more valued).
+    tries=    : Maximum number of random restarts on a single strategy before switching (Default 3)
                 Higher values will make optimizer run longer, more chances to find better solution.
  -P#          : generated sample compressibility 
  -t#          : Caps runtime of operation in seconds (default : 99999 seconds (about 27 hours )) 
diff --git a/tests/paramgrill.c b/tests/paramgrill.c
index 47e0a9421..193b2ed20 100644
--- a/tests/paramgrill.c
+++ b/tests/paramgrill.c
@@ -139,7 +139,7 @@ static BMK_result_t g_lvltarget;
 static int g_optmode = 0;
 
 static U32 g_speedMultiplier = 1;
-static U32 g_ratioMultiplier = 1;
+static U32 g_ratioMultiplier = 5;
 
 /* g_mode? */
 
@@ -382,7 +382,7 @@ typedef struct {
 } contexts_t;
 
 /*-*******************************************************
-*  From Paramgrill
+*  From bench.c
 *********************************************************/
 
 static void BMK_initCCtx(ZSTD_CCtx* ctx, 
@@ -411,7 +411,6 @@ static void BMK_initCCtx(ZSTD_CCtx* ctx,
     ZSTD_CCtx_loadDictionary(ctx, dictBuffer, dictBufferSize);
 }
 
-
 static void BMK_initDCtx(ZSTD_DCtx* dctx, 
     const void* dictBuffer, const size_t dictBufferSize) {
     ZSTD_DCtx_reset(dctx);
@@ -503,7 +502,7 @@ static size_t local_defaultDecompress(
 }
 
 /*-*******************************************************
-*  From Paramgrill End
+*  From bench.c End
 *********************************************************/
 
 static void freeBuffers(const buffers_t b) {
@@ -527,23 +526,25 @@ static void freeBuffers(const buffers_t b) {
     free(b.resSizes);
 }
 
-/* allocates buffer's arguments. returns success / failuere */
-static int createBuffers(buffers_t* buff, const char* const * const fileNamesTable, 
-                          const size_t nbFiles)
+/* srcBuffer will be freed by freeBuffers now */
+static int createBuffersFromMemory(buffers_t* buff, void * srcBuffer, size_t nbFiles,
+    const size_t* fileSizes)
 {
-    size_t pos = 0;
-    size_t n;
-    U64 const totalSizeToLoad = UTIL_getTotalFileSize(fileNamesTable, (U32)nbFiles);
-    const size_t benchedSize = MIN(BMK_findMaxMem(totalSizeToLoad * 3) / 3, totalSizeToLoad);
-    const size_t blockSize = g_blockSize ? g_blockSize : totalSizeToLoad;
-    U32 const maxNbBlocks = (U32) ((totalSizeToLoad + (blockSize-1)) / MAX(blockSize, 1)) + (U32)nbFiles;
-    U32 blockNb = 0;
+    size_t pos = 0, n, blockSize;
+    U32 maxNbBlocks, blockNb = 0;
+    buff->srcSize = 0;
+    for(n = 0; n < nbFiles; n++) {
+        buff->srcSize += fileSizes[n];
+    }
 
-    if(!totalSizeToLoad || !benchedSize) {
-        DISPLAY("Nothing to Bench\n");
+    if(buff->srcSize == 0) {
+        DISPLAY("No data to bench\n");
         return 1;
     }
-    
+
+    blockSize = g_blockSize ? g_blockSize : buff->srcSize;
+    maxNbBlocks = (U32) ((buff->srcSize + (blockSize-1)) / blockSize) + (U32)nbFiles;
+
     buff->srcPtrs = (const void**)calloc(maxNbBlocks, sizeof(void*));
     buff->srcSizes = (size_t*)malloc(maxNbBlocks * sizeof(size_t));
 
@@ -560,18 +561,62 @@ static int createBuffers(buffers_t* buff, const char* const * const fileNamesTab
         return 1;
     }
 
-
-    buff->srcBuffer = malloc(benchedSize);
+    buff->srcBuffer = srcBuffer;
     buff->srcPtrs[0] = (const void*)buff->srcBuffer;
-    buff->dstPtrs[0] = malloc(ZSTD_compressBound(benchedSize) + (maxNbBlocks * 1024));
-    buff->resPtrs[0] = malloc(benchedSize);
+    buff->dstPtrs[0] = malloc(ZSTD_compressBound(buff->srcSize) + (maxNbBlocks * 1024));
+    buff->resPtrs[0] = malloc(buff->srcSize);
 
-    if(!buff->srcPtrs[0] || !buff->dstPtrs[0] || !buff->resPtrs[0]) {
+    if(!buff->dstPtrs[0] || !buff->resPtrs[0]) {
         DISPLAY("alloc error\n");
         freeBuffers(*buff);
         return 1;
     }
 
+    for(n = 0; n < nbFiles; n++) {
+        size_t pos_end = pos + fileSizes[n];
+        for(; pos < pos_end; blockNb++) {
+            buff->srcPtrs[blockNb] = (const void*)((char*)srcBuffer + pos);
+            buff->srcSizes[blockNb] = blockSize;
+            pos += blockSize;
+        }
+
+        if(fileSizes[n] > 0) { buff->srcSizes[blockNb - 1] = ((fileSizes[n] - 1) % blockSize) + 1; }
+        pos = pos_end;
+    }
+
+    buff->dstCapacities[0] = ZSTD_compressBound(buff->srcSizes[0]);
+    buff->dstSizes[0] = buff->dstCapacities[0];
+    buff->resSizes[0] = buff->srcSizes[0];
+
+    for(n = 1; n < blockNb; n++) {
+        buff->dstPtrs[n] = ((char*)buff->dstPtrs[n-1]) + buff->dstCapacities[n-1];
+        buff->resPtrs[n] = ((char*)buff->resPtrs[n-1]) + buff->resSizes[n-1];
+        buff->dstCapacities[n] = ZSTD_compressBound(buff->srcSizes[n]);
+        buff->dstSizes[n] = buff->dstCapacities[n];
+        buff->resSizes[n] = buff->srcSizes[n];
+    }
+
+    buff->nbBlocks = blockNb;
+
+    return 0;
+} 
+
+/* allocates buffer's arguments. returns success / failuere */
+static int createBuffers(buffers_t* buff, const char* const * const fileNamesTable, 
+                          size_t nbFiles) {
+    size_t pos = 0;
+    size_t n;
+    size_t totalSizeToLoad = UTIL_getTotalFileSize(fileNamesTable, (U32)nbFiles);
+    size_t benchedSize = MIN(BMK_findMaxMem(totalSizeToLoad * 3) / 3, totalSizeToLoad);
+    size_t* fileSizes = calloc(sizeof(size_t), nbFiles);
+    void* srcBuffer = malloc(benchedSize);
+    int ret = 0;  
+
+
+    if(!fileSizes || !srcBuffer) {
+        return 1;
+    }
+
     for(n = 0; n < nbFiles; n++) {
         FILE* f;
         U64 fileSize = UTIL_getFileSize(fileNamesTable[n]);
@@ -586,62 +631,35 @@ static int createBuffers(buffers_t* buff, const char* const * const fileNamesTab
         f = fopen(fileNamesTable[n], "rb");
         if (f==NULL) {
             DISPLAY("impossible to open file %s\n", fileNamesTable[n]);
-            freeBuffers(*buff);
+            free(fileSizes);
+            free(srcBuffer);
             fclose(f);
             return 10;
         }
 
         DISPLAY("Loading %s...       \r", fileNamesTable[n]);
 
-        if (fileSize + pos > benchedSize) fileSize = benchedSize - pos, n=nbFiles;   /* buffer too small - stop after this file */
+        if (fileSize + pos > benchedSize) fileSize = benchedSize - pos, nbFiles=n;   /* buffer too small - stop after this file */
         {
-            char* buffer = (char*)(buff->srcBuffer); 
+            char* buffer = (char*)(srcBuffer); 
             size_t const readSize = fread((buffer)+pos, 1, (size_t)fileSize, f);
-            size_t blocked = 0;
-            while(blocked < readSize) {
-                buff->srcPtrs[blockNb] = (const void*)((buffer) + (pos + blocked));
-                buff->srcSizes[blockNb] = blockSize;
-                blocked += blockSize;
-                blockNb++;
-            }
-            if(readSize > 0) { buff->srcSizes[blockNb - 1] = ((readSize - 1) % blockSize) + 1; }
 
-            if (readSize != (size_t)fileSize) {
+            if (readSize != (size_t)fileSize) { /* should we accept partial read? */
                 DISPLAY("could not read %s", fileNamesTable[n]);
-                freeBuffers(*buff);
-                fclose(f);
+                free(fileSizes);
+                free(srcBuffer);
                 return 1;
             }
 
+            fileSizes[n] = readSize;
             pos += readSize;
-
         }
         fclose(f);
     }
 
-    if(!blockNb) {
-        DISPLAY("Failed to load any files\n");
-        freeBuffers(*buff);
-        return 1;
-    }
-
-    buff->dstCapacities[0] = ZSTD_compressBound(buff->srcSizes[0]);
-    buff->dstSizes[0] = buff->dstCapacities[0];
-    buff->resSizes[0] = buff->srcSizes[0];
-
-    for(n = 1; n < blockNb; n++) {
-        buff->dstPtrs[n] = ((char*)buff->dstPtrs[n-1]) + buff->dstCapacities[n-1];
-        buff->resPtrs[n] = ((char*)buff->resPtrs[n-1]) + buff->resSizes[n-1];
-        buff->dstCapacities[n] = ZSTD_compressBound(buff->srcSizes[n]);
-        buff->dstSizes[n] = buff->dstCapacities[n];
-        buff->resSizes[n] = buff->srcSizes[n];
-    }
-    buff->srcSize = pos;
-    buff->nbBlocks = blockNb;
-
-    if (pos == 0) { DISPLAY("\nno data to bench\n"); return 1; }
-
-    return 0;
+    ret = createBuffersFromMemory(buff, srcBuffer, nbFiles, fileSizes);
+    free(fileSizes);
+    return ret;
 }
 
 static void freeContexts(const contexts_t ctx) {
@@ -1611,7 +1629,7 @@ static void BMK_benchFullTable(buffers_t buf, contexts_t ctx, const size_t maxBl
         BMK_init_level_constraints(g_target * (1 MB));
     } else {
         /* baseline config for level 1 */
-        ZSTD_compressionParameters const l1params = ZSTD_getCParams(1, maxBlockSize, ctx.dictSize); //is dictionary ever even useful here? 
+        ZSTD_compressionParameters const l1params = ZSTD_getCParams(1, maxBlockSize, ctx.dictSize);
         BMK_result_t testResult;
         BMK_benchParam(&testResult, buf, ctx, l1params);
         BMK_init_level_constraints((int)((testResult.cSpeed * 31) / 32));
@@ -1641,82 +1659,6 @@ static void BMK_benchFullTable(buffers_t buf, contexts_t ctx, const size_t maxBl
     fclose(f);
 }
 
-static int benchSample(void)
-{
-    const char* const name = "Sample 10MB";
-    size_t const benchedSize = 10 MB;
-    U32 blockSize = g_blockSize ? g_blockSize : benchedSize;
-    U32 const maxNbBlocks = (U32) ((benchedSize + (blockSize-1)) / blockSize) + 1;
-    size_t splitSize = 0;
-
-    buffers_t buf;
-    contexts_t ctx;
-
-    buf.srcPtrs = (const void**)calloc(maxNbBlocks, sizeof(void*));
-    buf.dstPtrs = (void**)calloc(maxNbBlocks, sizeof(void*));
-    buf.resPtrs = (void**)calloc(maxNbBlocks, sizeof(void*));
-    buf.srcSizes = (size_t*)malloc(maxNbBlocks * sizeof(size_t));
-    buf.dstSizes = (size_t*)malloc(maxNbBlocks * sizeof(size_t));
-    buf.dstCapacities = (size_t*)malloc(maxNbBlocks * sizeof(size_t));
-    buf.resSizes = (size_t*)malloc(maxNbBlocks * sizeof(size_t));
-    buf.srcSize = benchedSize;
-
-    if(!buf.srcPtrs || !buf.dstPtrs || !buf.resPtrs || !buf.srcSizes || !buf.dstSizes || !buf.dstCapacities || !buf.resSizes) {
-        DISPLAY("Allocation Error\n");
-        freeBuffers(buf);
-        return 1;
-    } 
-
-    buf.srcBuffer = malloc(benchedSize);
-    buf.srcPtrs[0] = (const void*)buf.srcBuffer;
-    buf.dstPtrs[0] = malloc(ZSTD_compressBound(benchedSize) + 1024 * maxNbBlocks);
-    buf.resPtrs[0] = malloc(benchedSize);
-
-    if(!buf.srcPtrs[0] || !buf.dstPtrs[0] || !buf.resPtrs[0]) {
-        DISPLAY("Allocation Error\n");
-        freeBuffers(buf);
-        return 1;
-    }
-
-
-    splitSize = MIN(benchedSize, blockSize);
-    buf.srcSizes[0] = splitSize;
-    buf.dstCapacities[0] = ZSTD_compressBound(splitSize);
-    buf.resSizes[0] = splitSize;
-
-    for(buf.nbBlocks = 1; splitSize < benchedSize; buf.nbBlocks++) {
-        const size_t i = buf.nbBlocks;
-        const size_t nextBlockSize = MIN(benchedSize - splitSize, blockSize);
-        buf.srcSizes[i] = nextBlockSize;
-        buf.dstCapacities[i] = ZSTD_compressBound(nextBlockSize);
-        buf.resSizes[i] = nextBlockSize;
-        buf.srcPtrs[i] = (const void*)(((const char*)buf.srcPtrs[i-1]) + buf.srcSizes[i-1]);
-        buf.dstPtrs[i] = (void*)(((char*)buf.dstPtrs[i-1]) + buf.dstSizes[i-1]);
-        buf.resPtrs[i] = (void*)(((char*)buf.resPtrs[i-1]) + buf.resSizes[i-1]);
-        splitSize += nextBlockSize;
-    }
-
-    if(createContexts(&ctx, NULL)) {
-        DISPLAY("Context Creation Error\n");
-        freeBuffers(buf);
-        return 1;
-    }
-
-    RDG_genBuffer(buf.srcBuffer, benchedSize, g_compressibility, 0.0, 0);
-
-    /* bench */
-    DISPLAY("\r%79s\r", "");
-    DISPLAY("using %s %i%%: \n", name, (int)(g_compressibility*100));
-
-    BMK_benchFullTable(buf, ctx, MIN(blockSize, benchedSize));
-
-    freeBuffers(buf);
-    freeContexts(ctx);
-
-    return 0;
-}
-
-
 static int benchOnce(buffers_t buf, contexts_t ctx) {
     BMK_result_t testResult;
 
@@ -1726,9 +1668,56 @@ static int benchOnce(buffers_t buf, contexts_t ctx) {
     }
 
     BMK_printWinner(stdout, CUSTOM_LEVEL, testResult, g_params, buf.srcSize);
+
     return 0;
 }
 
+static int benchSample(void)
+{
+    const char* const name = "Sample 10MB";
+    size_t const benchedSize = 10 MB;
+    U32 blockSize = g_blockSize ? g_blockSize : benchedSize;
+    void* srcBuffer = malloc(benchedSize);
+    int ret = 0;
+
+    buffers_t buf;
+    contexts_t ctx;
+    
+    if(srcBuffer == NULL) {
+        DISPLAY("Out of Memory\n");
+        return 2;
+    }
+
+    RDG_genBuffer(srcBuffer, benchedSize, g_compressibility, 0.0, 0);
+
+    if(createBuffersFromMemory(&buf, srcBuffer, 1, &benchedSize)) {
+        DISPLAY("Buffer Creation Error\n");
+        free(srcBuffer);
+        return 3;
+    }
+
+    if(createContexts(&ctx, NULL)) {
+        DISPLAY("Context Creation Error\n");
+        freeBuffers(buf);
+        return 1;
+    }
+
+    /* bench */
+    DISPLAY("\r%79s\r", "");
+    DISPLAY("using %s %i%%: \n", name, (int)(g_compressibility*100));
+
+    if(g_singleRun) {
+        ret = benchOnce(buf, ctx);
+    } else {
+        BMK_benchFullTable(buf, ctx, MIN(blockSize, benchedSize));
+    }
+
+    freeBuffers(buf);
+    freeContexts(ctx);
+
+    return ret;
+}
+
 /* benchFiles() :
  * note: while this function takes a table of filenames,
  * in practice, only the first filename will be used */
@@ -1776,6 +1765,7 @@ int benchFiles(const char** fileNamesTable, int nbFiles, const char* dictFileNam
 
 /* Benchmarking which stops when we are sufficiently sure the solution is infeasible / worse than the winner */
 #define VARIANCE 1.2 
+#define HIGH_VARIANCE 100.0
 static int allBench(BMK_result_t* resultPtr,
                 const buffers_t buf, const contexts_t ctx,
                 const ZSTD_compressionParameters cParams,
@@ -1784,7 +1774,7 @@ static int allBench(BMK_result_t* resultPtr,
     BMK_return_t benchres;
     BMK_result_t resultMax;
     U64 loopDurationC = 0, loopDurationD = 0;
-    double uncertaintyConstantC, uncertaintyConstantD;
+    double uncertaintyConstantC = 3., uncertaintyConstantD = 3.;
     double winnerRS;
 
     /* initial benchmarking, gives exact ratio and memory, warms up future runs */
@@ -1802,18 +1792,12 @@ static int allBench(BMK_result_t* resultPtr,
     /* calculate uncertainty in compression / decompression runs */
     if(benchres.result.cSpeed) {
         loopDurationC = ((buf.srcSize * TIMELOOP_NANOSEC) / benchres.result.cSpeed); 
-        uncertaintyConstantC = ((loopDurationC + (double)(2 * g_clockGranularity))/loopDurationC) * VARIANCE; 
-    } else {
-        loopDurationC = 0;
-        uncertaintyConstantC = 3;
+        uncertaintyConstantC = ((loopDurationC + (double)(2 * g_clockGranularity))/loopDurationC); 
     }
 
     if(benchres.result.dSpeed) {
         loopDurationD = ((buf.srcSize * TIMELOOP_NANOSEC) / benchres.result.dSpeed); 
-        uncertaintyConstantD = ((loopDurationD + (double)(2 * g_clockGranularity))/loopDurationD) * VARIANCE;  
-    } else {
-        loopDurationD = 0;
-        uncertaintyConstantD = 3;
+        uncertaintyConstantD = ((loopDurationD + (double)(2 * g_clockGranularity))/loopDurationD);  
     }
 
     /* anything with worse ratio in feas is definitely worse, discard */
@@ -1841,8 +1825,8 @@ static int allBench(BMK_result_t* resultPtr,
 
     /* optimistic assumption of benchres.result */
     resultMax = benchres.result;
-    resultMax.cSpeed *= uncertaintyConstantC;
-    resultMax.dSpeed *= uncertaintyConstantD;
+    resultMax.cSpeed *= uncertaintyConstantC * VARIANCE;
+    resultMax.dSpeed *= uncertaintyConstantD * VARIANCE;
 
     /* disregard infeasible results in feas mode */
     /* disregard if resultMax < winner in infeas mode */
@@ -2125,7 +2109,7 @@ static int nextStrategy(const int currentStrategy, const int bestStrategy) {
  * cLevel - compression level to exceed (all solutions must be > lvl in cSpeed + ratio)
  */
 
-static int g_maxTries = 5;
+static int g_maxTries = 3;
 #define TRY_DECAY 1
 
 static int optimizeForSize(const char* const * const fileNamesTable, const size_t nbFiles, const char* dictFileName, constraint_t target, ZSTD_compressionParameters paramTarget, int cLevel)
@@ -2140,8 +2124,6 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_
     contexts_t ctx;
     buffers_t buf;
 
-    g_time = UTIL_getTime();
-
     /* Init */
     if(!cParamValid(paramTarget)) {
         return 1;
@@ -2234,6 +2216,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_
         BMK_printWinnerOpt(stdout, cLevel, winner.result, winner.params, target, buf.srcSize);
     }
 
+    /* Don't want it to return anything worse than the best known result */
     if(g_singleRun) {
         BMK_result_t res;
         g_params = ZSTD_adjustCParams(maskParams(ZSTD_getCParams(cLevel, maxBlockSize, ctx.dictSize), g_params), maxBlockSize, ctx.dictSize);
@@ -2450,6 +2433,8 @@ int main(int argc, const char** argv)
 
     assert(argc>=1);   /* for exename */
 
+    g_time = UTIL_getTime();
+
     /* Welcome message */
     DISPLAY(WELCOME_MESSAGE);
 

From e3c679484aed822ec5574ae426a77490966b35aa Mon Sep 17 00:00:00 2001
From: George Lu 
Date: Mon, 6 Aug 2018 16:52:17 -0700
Subject: [PATCH 161/372] Add Time Checks Fix double -> U64 display

---
 tests/paramgrill.c | 38 ++++++++++++++++++++------------------
 1 file changed, 20 insertions(+), 18 deletions(-)

diff --git a/tests/paramgrill.c b/tests/paramgrill.c
index 193b2ed20..af4459187 100644
--- a/tests/paramgrill.c
+++ b/tests/paramgrill.c
@@ -17,7 +17,6 @@
 #include      /* fprintf, fopen, ftello64 */
 #include     /* strcmp */
 #include       /* log */
-#include 
 #include 
 
 #include "mem.h"
@@ -96,6 +95,9 @@ typedef enum {
 #define STRT_RANGE (ZSTD_btultra - ZSTD_fast + 1)
 /* TLEN_RANGE picked manually */
 
+#define CHECKTIME(r) { if(BMK_timeSpan(g_time) > g_timeLimit_s) { DEBUGOUTPUT("Time Limit Reached\n"); return r; } }
+#define CHECKTIMEGT(ret, val, _gototag) {if(BMK_timeSpan(g_time) > g_timeLimit_s) { DEBUGOUTPUT("Time Limit Reached\n"); ret = val; goto _gototag; } }
+
 static const int rangetable[NUM_PARAMS] = { WLOG_RANGE, CLOG_RANGE, HLOG_RANGE, SLOG_RANGE, SLEN_RANGE, TLEN_RANGE, STRT_RANGE };
 static const U32 tlen_table[TLEN_RANGE] = { 0, 1, 2, 4, 6, 8, 12, 16, 24, 32, 48, 64, 96, 128, 256, 512, 999 };
 /*-************************************
@@ -104,7 +106,7 @@ static const U32 tlen_table[TLEN_RANGE] = { 0, 1, 2, 4, 6, 8, 12, 16, 24, 32, 48
 
 typedef BYTE U8;
 
-static double g_grillDuration_s = 99999;   /* about 27 hours */
+static U32 g_timeLimit_s = 99999;   /* about 27 hours */
 static U32 g_nbIterations = NBLOOPS;
 static double g_compressibility = COMPRESSIBILITY_DEFAULT;
 static U32 g_blockSize = 0;
@@ -166,7 +168,7 @@ void BMK_SetNbIterations(int nbLoops)
 *********************************************************/
 
 /* accuracy in seconds only, span can be multiple years */
-static double BMK_timeSpan(time_t tStart) { return difftime(time(NULL), tStart); }
+static U32 BMK_timeSpan(UTIL_time_t tStart) { return (U32)(UTIL_clockSpanMicro(tStart) / 1000000ULL); }
 
 static size_t BMK_findMaxMem(U64 requiredMem)
 {
@@ -1252,8 +1254,7 @@ static int sanitizeVarArray(varInds_t* varNew, const int varLength, const varInd
         if( !((varArray[i] == clog_ind && strat == ZSTD_fast)
             || (varArray[i] == slog_ind && strat == ZSTD_fast)
             || (varArray[i] == slog_ind && strat == ZSTD_dfast) 
-            || (varArray[i] == tlen_ind && strat != ZSTD_btopt && strat != ZSTD_btultra && strat != ZSTD_fast)
-            /* || varArray[i] == strt_ind */ )) {
+            || (varArray[i] == tlen_ind && strat != ZSTD_btopt && strat != ZSTD_btultra && strat != ZSTD_fast))) {
             varNew[j] = varArray[i];
             j++;
         }
@@ -1645,10 +1646,10 @@ static void BMK_benchFullTable(buffers_t buf, contexts_t ctx, const size_t maxBl
     BMK_printWinners(f, winners, buf.srcSize);
 
     /* start tests */
-    {   const time_t grillStart = time(NULL);
+    {   const UTIL_time_t grillStart = UTIL_getTime();
         do {
             BMK_selectRandomStart(f, winners, buf, ctx);
-        } while (BMK_timeSpan(grillStart) < g_grillDuration_s);
+        } while (BMK_timeSpan(grillStart) < g_timeLimit_s);
     }
 
     /* end summary */
@@ -1893,7 +1894,6 @@ static int benchMemo(BMK_result_t* resultPtr,
     return res;
 }
 
-
 /* One iteration of hill climbing. Specifically, it first tries all 
  * valid parameter configurations w/ manhattan distance 1 and picks the best one
  * failing that, it progressively tries candidates further and further away (up to #dim + 2)
@@ -1944,6 +1944,7 @@ static winnerInfo_t climbOnce(const constraint_t target,
              /* all dist-1 candidates */
             for(i = 0; i < varLen; i++) {
                 for(offset = -1; offset <= 1; offset += 2) {
+                    CHECKTIME(winnerInfo);
                     candidateInfo.params = cparam;
                     paramVaryOnce(varArray[i], offset, &candidateInfo.params); 
                     
@@ -1953,8 +1954,7 @@ static winnerInfo_t climbOnce(const constraint_t target,
                             strat = candidateInfo.params.strategy;
                             varLenNew = sanitizeVarArray(varNew, varLen, varArray, strat);
                         }
-                        res = benchMemo(&candidateInfo.result,
-                            buf, ctx,
+                        res = benchMemo(&candidateInfo.result, buf, ctx,
                             sanitizeParams(candidateInfo.params), target, &winnerInfo.result, memoTableArray[strat],
                             varNew, varLenNew, feas);
                         if(res == BETTER_RESULT) { /* synonymous with better when called w/ infeasibleBM */
@@ -1976,6 +1976,7 @@ static winnerInfo_t climbOnce(const constraint_t target,
             for(dist = 2; dist < varLen + 2; dist++) { /* varLen is # dimensions */
                 for(i = 0; i < (1 << varLen) / varLen + 2; i++) {
                     int res;
+                    CHECKTIME(winnerInfo);
                     candidateInfo.params = cparam;
                     /* param error checking already done here */
                     paramVariation(&candidateInfo.params, varArray, varLen, dist);
@@ -1985,8 +1986,7 @@ static winnerInfo_t climbOnce(const constraint_t target,
                         varLenNew = sanitizeVarArray(varNew, varLen, varArray, strat);
                     }
 
-                    res = benchMemo(&candidateInfo.result,
-                        buf, ctx, 
+                    res = benchMemo(&candidateInfo.result, buf, ctx, 
                         sanitizeParams(candidateInfo.params), target, &winnerInfo.result, memoTableArray[strat],
                         varNew, varLenNew, feas);
                     if(res == BETTER_RESULT) { /* synonymous with better in this case*/
@@ -2058,7 +2058,7 @@ static winnerInfo_t optimizeFixedStrategy(
             BMK_printWinnerOpt(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, target, buf.srcSize);
             i = 0;
         }
-
+        CHECKTIME(winnerInfo);
         i++;
     }
     return winnerInfo;
@@ -2123,6 +2123,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_
     size_t maxBlockSize = 0;
     contexts_t ctx;
     buffers_t buf;
+    g_time = UTIL_getTime();
 
     /* Init */
     if(!cParamValid(paramTarget)) {
@@ -2174,7 +2175,6 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_
          allMT = createMemoTableArray(paramTarget, target, varArray, varLen, maxBlockSize);
     }
    
-
     if(!allMT) {
         DISPLAY("MemoTable Init Error\n");
         ret = 2;
@@ -2267,6 +2267,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_
                         winner.params = CParams;
                     }
 
+                    CHECKTIMEGT(ret, 0, _cleanUp); /* if pass time limit, stop */
                     /* if the current params are too slow, just stop. */
                     if(target.cSpeed > candidate.cSpeed * 3 / 2) { break; }
                 }
@@ -2290,6 +2291,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_
                     if(compareResultLT(winner.result, w1.result, target, buf.srcSize)) {
                         winner = w1;
                     }
+                    CHECKTIMEGT(ret, 0, _cleanUp);
                 }
 
                 while(st && tries > 0) {
@@ -2307,6 +2309,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_
                         st = nextStrategy(st, bestStrategy);
                         tries -= TRY_DECAY;
                     }
+                    CHECKTIMEGT(ret, 0, _cleanUp);
                 }
             } else {
                 winner = optimizeFixedStrategy(buf, ctx, target, paramTarget, paramTarget.strategy, 
@@ -2388,7 +2391,7 @@ static int usage_advanced(void)
     DISPLAY( " -S           : Single run \n");
     DISPLAY( " --zstd       : Single run, parameter selection same as zstdcli \n");
     DISPLAY( " -P#          : generated sample compressibility (default : %.1f%%) \n", COMPRESSIBILITY_DEFAULT * 100);
-    DISPLAY( " -t#          : Caps runtime of operation in seconds (default : %u seconds (%.1f hours)) \n", (U32)g_grillDuration_s, g_grillDuration_s / 3600);
+    DISPLAY( " -t#          : Caps runtime of operation in seconds (default : %u seconds (%.1f hours)) \n", g_timeLimit_s, (double)g_timeLimit_s / 3600);
     DISPLAY( " -v           : Prints Benchmarking output\n");
     DISPLAY( " -D           : Next argument dictionary file\n");
     DISPLAY( " -s           : Seperate Files\n");
@@ -2433,8 +2436,6 @@ int main(int argc, const char** argv)
 
     assert(argc>=1);   /* for exename */
 
-    g_time = UTIL_getTime();
-
     /* Welcome message */
     DISPLAY(WELCOME_MESSAGE);
 
@@ -2580,7 +2581,7 @@ int main(int argc, const char** argv)
                     /* caps runtime (in seconds) */
                 case 't':
                     argument++;
-                    g_grillDuration_s = (double)readU32FromChar(&argument);
+                    g_timeLimit_s = readU32FromChar(&argument);
                     break;
 
                 case 's':
@@ -2609,6 +2610,7 @@ int main(int argc, const char** argv)
         /* first provided filename is input */
         if (!input_filename) { input_filename=argument; filenamesStart=i; continue; }
     }
+
     if (filenamesStart==0) {
         if (g_optimizer) {
             DISPLAY("Optimizer Expects File\n");

From 3f2d024dca7723b4235618d99ce755287e530472 Mon Sep 17 00:00:00 2001
From: George Lu 
Date: Thu, 9 Aug 2018 10:42:35 -0700
Subject: [PATCH 162/372] forceAttachDict

---
 tests/paramgrill.c | 746 ++++++++++++++++++++++++---------------------
 1 file changed, 397 insertions(+), 349 deletions(-)

diff --git a/tests/paramgrill.c b/tests/paramgrill.c
index af4459187..aab46ede5 100644
--- a/tests/paramgrill.c
+++ b/tests/paramgrill.c
@@ -66,22 +66,10 @@ static const int g_maxNbVariations = 64;
 #define CUSTOM_LEVEL 99
 #define BASE_CLEVEL 1
 
-/* indices for each of the variables */
-typedef enum {
-    wlog_ind = 0,
-    clog_ind = 1,
-    hlog_ind = 2,
-    slog_ind = 3,
-    slen_ind = 4,
-    tlen_ind = 5,
-    strt_ind = 6
-} varInds_t;
-
-#define NUM_PARAMS 7
-/* just don't use strategy as a param. */ 
-
 #undef ZSTD_WINDOWLOG_MAX
 #define ZSTD_WINDOWLOG_MAX 27 //no long range stuff for now. 
+#define FADT_MIN 0
+#define FADT_MAX ((U32)-1)
 
 #define ZSTD_TARGETLENGTH_MIN 0 
 #define ZSTD_TARGETLENGTH_MAX 999
@@ -93,13 +81,146 @@ typedef enum {
 #define SLEN_RANGE (ZSTD_SEARCHLENGTH_MAX - ZSTD_SEARCHLENGTH_MIN + 1)
 #define TLEN_RANGE 17
 #define STRT_RANGE (ZSTD_btultra - ZSTD_fast + 1)
-/* TLEN_RANGE picked manually */
+#define FADT_RANGE 3
 
 #define CHECKTIME(r) { if(BMK_timeSpan(g_time) > g_timeLimit_s) { DEBUGOUTPUT("Time Limit Reached\n"); return r; } }
 #define CHECKTIMEGT(ret, val, _gototag) {if(BMK_timeSpan(g_time) > g_timeLimit_s) { DEBUGOUTPUT("Time Limit Reached\n"); ret = val; goto _gototag; } }
 
-static const int rangetable[NUM_PARAMS] = { WLOG_RANGE, CLOG_RANGE, HLOG_RANGE, SLOG_RANGE, SLEN_RANGE, TLEN_RANGE, STRT_RANGE };
+#define PARAM_UNSET ((U32)-2) /* can't be -1 b/c fadt */
+
+/*-************************************
+*  Setup for Adding new params
+**************************************/
+
+/* indices for each of the variables */
+typedef enum {
+    wlog_ind = 0,
+    clog_ind = 1,
+    hlog_ind = 2,
+    slog_ind = 3,
+    slen_ind = 4,
+    tlen_ind = 5,
+    strt_ind = 6,
+    fadt_ind = 7, /* forceAttachDict */
+    NUM_PARAMS = 8
+} varInds_t;
+
+/* maximum value of parameters */
+static const U32 mintable[NUM_PARAMS] = 
+        { ZSTD_WINDOWLOG_MIN, ZSTD_CHAINLOG_MIN, ZSTD_HASHLOG_MIN, ZSTD_SEARCHLOG_MIN, ZSTD_SEARCHLENGTH_MIN, ZSTD_TARGETLENGTH_MIN, ZSTD_fast, FADT_MIN };
+
+/* minimum value of parameters */
+static const U32 maxtable[NUM_PARAMS] = 
+        { ZSTD_WINDOWLOG_MAX, ZSTD_CHAINLOG_MAX, ZSTD_HASHLOG_MAX, ZSTD_SEARCHLOG_MAX, ZSTD_SEARCHLENGTH_MAX, ZSTD_TARGETLENGTH_MAX, ZSTD_btultra, FADT_MAX };
+
+/* # of values parameters can take on */
+static const U32 rangetable[NUM_PARAMS] = 
+        { WLOG_RANGE, CLOG_RANGE, HLOG_RANGE, SLOG_RANGE, SLEN_RANGE, TLEN_RANGE, STRT_RANGE, FADT_RANGE };
+
+static const ZSTD_cParameter cctxSetParamTable[NUM_PARAMS] = 
+        { ZSTD_p_windowLog, ZSTD_p_chainLog, ZSTD_p_hashLog, ZSTD_p_searchLog, ZSTD_p_minMatch, ZSTD_p_targetLength, ZSTD_p_compressionStrategy, ZSTD_p_forceAttachDict };
+
+static const char* g_paramNames[NUM_PARAMS] = 
+        { "windowLog", "chainLog", "hashLog","searchLog", "searchLength", "targetLength", "strategy", "forceAttachDict"};
+
+
 static const U32 tlen_table[TLEN_RANGE] = { 0, 1, 2, 4, 6, 8, 12, 16, 24, 32, 48, 64, 96, 128, 256, 512, 999 };
+/* maps value from 0 to rangetable[param] - 1 to valid paramvalue */
+static U32 rangeMap(varInds_t param, U32 ind) {
+    ind = MIN(ind, rangetable[param] - 1);
+    switch(param) {
+        case tlen_ind:
+            return tlen_table[ind];
+        case fadt_ind: /* 0, 1, 2 -> -1, 0, 1 */
+            return ind - 1;
+        case wlog_ind: /* using default: triggers -Wswitch-enum */
+        case clog_ind:
+        case hlog_ind:
+        case slog_ind:
+        case slen_ind:
+        case strt_ind:
+            return mintable[param] + ind;
+        case NUM_PARAMS:
+            return (U32)-1;
+    }
+    return 0; /* should never happen, stop compiler warnings */
+}
+
+/* inverse of rangeMap */
+static U32 invRangeMap(varInds_t param, U32 value) {
+    value = MIN(MAX(mintable[param], value), maxtable[param]);
+    switch(param) {
+        case tlen_ind: /* bin search */
+        {
+            int lo = 0;
+            int hi = TLEN_RANGE;
+            while(lo < hi) {
+                int mid = (lo + hi) / 2;
+                if(tlen_table[mid] < value) {
+                    lo = mid + 1;
+                } if(tlen_table[mid] == value) {
+                    return mid;
+                } else {
+                    hi = mid;
+                }
+            }
+            return lo;    
+        }
+        case fadt_ind:
+            return value + 1;
+        case wlog_ind:
+        case clog_ind:
+        case hlog_ind:
+        case slog_ind:
+        case slen_ind:
+        case strt_ind:
+            return value - mintable[param];
+        case NUM_PARAMS:
+            return (U32)-1;
+    }
+    return 0; /* should never happen, stop compiler warnings */
+}
+
+typedef struct {
+    U32 vals[NUM_PARAMS];
+} paramValues_t;
+
+//TODO: unset -> 0?
+static ZSTD_compressionParameters pvalsToCParams(paramValues_t p) {
+    ZSTD_compressionParameters c;
+    c.windowLog = p.vals[wlog_ind];
+    c.chainLog = p.vals[clog_ind];
+    c.hashLog = p.vals[hlog_ind];
+    c.searchLog = p.vals[slog_ind];
+    c.searchLength = p.vals[slen_ind];
+    c.targetLength = p.vals[tlen_ind];
+    c.strategy = p.vals[strt_ind];
+    /* no forceAttachDict */
+    return c;
+}
+
+/* 0 = auto for fadt */
+static paramValues_t cParamsToPVals(ZSTD_compressionParameters c) {
+    paramValues_t p;
+    p.vals[wlog_ind] = c.windowLog;
+    p.vals[clog_ind] = c.chainLog;
+    p.vals[hlog_ind] = c.hashLog;
+    p.vals[slog_ind] = c.searchLog;
+    p.vals[slen_ind] = c.searchLength;
+    p.vals[tlen_ind] = c.targetLength;
+    p.vals[strt_ind] = c.strategy;
+    p.vals[fadt_ind] = 0;
+    return p;
+}
+
+/* equivalent of ZSTD_adjustCParams for paramValues_t */
+static paramValues_t adjustParams(paramValues_t p, size_t maxBlockSize, size_t dictSize) {
+    U32 fval = p.vals[fadt_ind];
+    p = cParamsToPVals(ZSTD_adjustCParams(pvalsToCParams(p), maxBlockSize, dictSize));
+    p.vals[fadt_ind] = fval;
+    return p;
+}
+
 /*-************************************
 *  Benchmark Parameters/Global Variables
 **************************************/
@@ -115,15 +236,20 @@ static U32 g_singleRun = 0;
 static U32 g_optimizer = 0;
 static U32 g_target = 0;
 static U32 g_noSeed = 0;
-static ZSTD_compressionParameters g_params; /* Initialized at the beginning of main w/ emptyParams() function */
+static paramValues_t g_params; /* Initialized at the beginning of main w/ emptyParams() function */
 static UTIL_time_t g_time; /* to be used to compare solution finding speeds to compare to original */
 
 
 typedef struct {
     BMK_result_t result;
-    ZSTD_compressionParameters params;
+    paramValues_t params;
 } winnerInfo_t;
 
+typedef struct {
+    BMK_result_t result;
+    ZSTD_compressionParameters params;
+} oldWinnerInfo_t;
+
 typedef struct {
     U32 cSpeed;  /* bytes / sec */
     U32 dSpeed;
@@ -242,8 +368,9 @@ static void findClockGranularity(void) {
     DEBUGOUTPUT("Granularity: %llu\n", (unsigned long long)g_clockGranularity);
 }
 
+/* allows zeros */
 #define CLAMPCHECK(val,min,max) {                     \
-    if (val && (((val)<(min)) | ((val)>(max)))) {     \
+    if (((val)<(min)) | ((val)>(max))) {              \
         DISPLAY("INVALID PARAMETER CONSTRAINTS\n");   \
         return 0;                                     \
 }   }
@@ -251,36 +378,53 @@ static void findClockGranularity(void) {
 
 /* Like ZSTD_checkCParams() but allows 0's */
 /* no check on targetLen? */
-static int cParamValid(ZSTD_compressionParameters paramTarget) {
-    CLAMPCHECK(paramTarget.hashLog, ZSTD_HASHLOG_MIN, ZSTD_HASHLOG_MAX);
-    CLAMPCHECK(paramTarget.searchLog, ZSTD_SEARCHLOG_MIN, ZSTD_SEARCHLOG_MAX);
-    CLAMPCHECK(paramTarget.searchLength, ZSTD_SEARCHLENGTH_MIN, ZSTD_SEARCHLENGTH_MAX);
-    CLAMPCHECK(paramTarget.windowLog, ZSTD_WINDOWLOG_MIN, ZSTD_WINDOWLOG_MAX);
-    CLAMPCHECK(paramTarget.chainLog, ZSTD_CHAINLOG_MIN, ZSTD_CHAINLOG_MAX);
-    if(paramTarget.targetLength > ZSTD_TARGETLENGTH_MAX) {
-        DISPLAY("INVALID PARAMETER CONSTRAINTS\n");
-        return 0;
-    }
-    if(paramTarget.strategy > ZSTD_btultra) {
-        DISPLAY("INVALID PARAMETER CONSTRAINTS\n");
-        return 0;
+static int paramValid(paramValues_t paramTarget) {
+    U32 i;
+    for(i = 0; i < NUM_PARAMS; i++) {
+        CLAMPCHECK(paramTarget.vals[i], mintable[i], maxtable[i]);
     }
+    //TODO: Strategy could be valid at 0 before, is that right?
     return 1;
 }
 
-static void cParamZeroMin(ZSTD_compressionParameters* paramTarget) {
-    paramTarget->windowLog = paramTarget->windowLog ? paramTarget->windowLog : ZSTD_WINDOWLOG_MIN;
-    paramTarget->searchLog = paramTarget->searchLog ? paramTarget->searchLog : ZSTD_SEARCHLOG_MIN;
-    paramTarget->chainLog = paramTarget->chainLog ? paramTarget->chainLog : ZSTD_CHAINLOG_MIN;
-    paramTarget->hashLog = paramTarget->hashLog ? paramTarget->hashLog : ZSTD_HASHLOG_MIN;
-    paramTarget->searchLength = paramTarget->searchLength ? paramTarget->searchLength : ZSTD_SEARCHLENGTH_MIN;
-    paramTarget->targetLength = paramTarget->targetLength ? paramTarget->targetLength : 0;
+//TODO: doesn't affect strategy?
+static paramValues_t cParamUnsetMin(paramValues_t paramTarget) {
+    varInds_t i;
+    for(i = 0; i < NUM_PARAMS; i++) {
+        if(paramTarget.vals[i] == PARAM_UNSET) {
+            paramTarget.vals[i] = mintable[i];
+        }
+    }
+    return paramTarget;
 }
 
-static void BMK_translateAdvancedParams(const ZSTD_compressionParameters params)
-{
-    DISPLAY("--zstd=windowLog=%u,chainLog=%u,hashLog=%u,searchLog=%u,searchLength=%u,targetLength=%u,strategy=%u \n",
-             params.windowLog, params.chainLog, params.hashLog, params.searchLog, params.searchLength, params.targetLength, (U32)(params.strategy));
+static void BMK_translateAdvancedParams(FILE* f, const paramValues_t params) {
+    U32 i;
+    fprintf(f,"--zstd=");
+    for(i = 0; i < NUM_PARAMS; i++) {
+        fprintf(f,"%s", g_paramNames[i]);
+        fprintf(f,"=%u", params.vals[i]);
+        if(i != NUM_PARAMS - 1) {
+            fprintf(f, ",");
+        }
+    }
+    fprintf(f, "\n");
+}
+
+
+static const char* g_stratName[ZSTD_btultra+1] = {
+                "(none)       ", "ZSTD_fast    ", "ZSTD_dfast   ",
+                "ZSTD_greedy  ", "ZSTD_lazy    ", "ZSTD_lazy2   ",
+                "ZSTD_btlazy2 ", "ZSTD_btopt   ", "ZSTD_btultra "};
+
+static void BMK_displayOneResult(FILE* f, winnerInfo_t res, size_t srcSize) {
+            res.params = cParamUnsetMin(res.params);
+            fprintf(f,"    {%3u,%3u,%3u,%3u,%3u,%3u,%3d, %s},  ",
+                res.params.vals[wlog_ind], res.params.vals[clog_ind], res.params.vals[hlog_ind], res.params.vals[slog_ind], res.params.vals[slen_ind],
+                res.params.vals[tlen_ind], (int)res.params.vals[fadt_ind], g_stratName[res.params.vals[strt_ind]]);
+            fprintf(f,
+            "   /* R:%5.3f at %5.1f MB/s - %5.1f MB/s */\n",
+            (double)srcSize / res.result.cSize, (double)res.result.cSpeed / (1 MB), (double)res.result.dSpeed / (1 MB));
 }
 
 /* checks results are feasible */
@@ -343,17 +487,16 @@ static constraint_t relaxTarget(constraint_t target) {
 *  Bench functions
 *********************************************************/
 
-const char* g_stratName[ZSTD_btultra+1] = {
-                "(none)       ", "ZSTD_fast    ", "ZSTD_dfast   ",
-                "ZSTD_greedy  ", "ZSTD_lazy    ", "ZSTD_lazy2   ",
-                "ZSTD_btlazy2 ", "ZSTD_btopt   ", "ZSTD_btultra "};
-
-static ZSTD_compressionParameters emptyParams(void) {
-    ZSTD_compressionParameters p = { 0, 0, 0, 0, 0, 0, (ZSTD_strategy)0 };
+static paramValues_t emptyParams(void) {
+    U32 i;
+    paramValues_t p;
+    for(i = 0; i < NUM_PARAMS; i++) {
+        p.vals[i] = PARAM_UNSET;
+    }
     return p;
 }
 
-static winnerInfo_t initWinnerInfo(ZSTD_compressionParameters p) {
+static winnerInfo_t initWinnerInfo(paramValues_t p) {
     winnerInfo_t w1;
     w1.result.cSpeed = 0.;
     w1.result.dSpeed = 0.;
@@ -389,27 +532,16 @@ typedef struct {
 
 static void BMK_initCCtx(ZSTD_CCtx* ctx, 
     const void* dictBuffer, const size_t dictBufferSize, const int cLevel, 
-    const ZSTD_compressionParameters* comprParams, const BMK_advancedParams_t* adv) {
+    const paramValues_t* comprParams) {
+    varInds_t i;
     ZSTD_CCtx_reset(ctx);
     ZSTD_CCtx_resetParameters(ctx);
-    if (adv->nbWorkers==1) {
-        ZSTD_CCtx_setParameter(ctx, ZSTD_p_nbWorkers, 0);
-    } else {
-        ZSTD_CCtx_setParameter(ctx, ZSTD_p_nbWorkers, adv->nbWorkers);
-    }
     ZSTD_CCtx_setParameter(ctx, ZSTD_p_compressionLevel, cLevel);
-    ZSTD_CCtx_setParameter(ctx, ZSTD_p_enableLongDistanceMatching, adv->ldmFlag);
-    ZSTD_CCtx_setParameter(ctx, ZSTD_p_ldmMinMatch, adv->ldmMinMatch);
-    ZSTD_CCtx_setParameter(ctx, ZSTD_p_ldmHashLog, adv->ldmHashLog);
-    ZSTD_CCtx_setParameter(ctx, ZSTD_p_ldmBucketSizeLog, adv->ldmBucketSizeLog);
-    ZSTD_CCtx_setParameter(ctx, ZSTD_p_ldmHashEveryLog, adv->ldmHashEveryLog);
-    ZSTD_CCtx_setParameter(ctx, ZSTD_p_windowLog, comprParams->windowLog);
-    ZSTD_CCtx_setParameter(ctx, ZSTD_p_hashLog, comprParams->hashLog);
-    ZSTD_CCtx_setParameter(ctx, ZSTD_p_chainLog, comprParams->chainLog);
-    ZSTD_CCtx_setParameter(ctx, ZSTD_p_searchLog, comprParams->searchLog);
-    ZSTD_CCtx_setParameter(ctx, ZSTD_p_minMatch, comprParams->searchLength);
-    ZSTD_CCtx_setParameter(ctx, ZSTD_p_targetLength, comprParams->targetLength);
-    ZSTD_CCtx_setParameter(ctx, ZSTD_p_compressionStrategy, comprParams->strategy);
+
+    for(i = 0; i < NUM_PARAMS; i++) {
+        if(comprParams->vals[i] != PARAM_UNSET)
+        ZSTD_CCtx_setParameter(ctx, cctxSetParamTable[i], comprParams->vals[i]);
+    }
     ZSTD_CCtx_loadDictionary(ctx, dictBuffer, dictBufferSize);
 }
 
@@ -424,13 +556,12 @@ typedef struct {
     const void* dictBuffer;
     size_t dictBufferSize;
     int cLevel;
-    const ZSTD_compressionParameters* comprParams;
-    const BMK_advancedParams_t* adv;
+    const paramValues_t* comprParams;
 } BMK_initCCtxArgs;
 
 static size_t local_initCCtx(void* payload) {
     const BMK_initCCtxArgs* ag = (const BMK_initCCtxArgs*)payload;
-    BMK_initCCtx(ag->ctx, ag->dictBuffer, ag->dictBufferSize, ag->cLevel, ag->comprParams, ag->adv);
+    BMK_initCCtx(ag->ctx, ag->dictBuffer, ag->dictBufferSize, ag->cLevel, ag->comprParams);
     return 0;
 }
 
@@ -507,10 +638,7 @@ static size_t local_defaultDecompress(
 *  From bench.c End
 *********************************************************/
 
-static void freeBuffers(const buffers_t b) {
-    if(b.srcPtrs != NULL) {
-        free(b.srcBuffer);
-    }
+static void freeNonSrcBuffers(const buffers_t b) {
     free(b.srcPtrs);
     free(b.srcSizes);
 
@@ -528,6 +656,13 @@ static void freeBuffers(const buffers_t b) {
     free(b.resSizes);
 }
 
+static void freeBuffers(const buffers_t b) {
+    if(b.srcPtrs != NULL) {
+        free(b.srcBuffer);
+    }
+    freeNonSrcBuffers(b);
+}
+
 /* srcBuffer will be freed by freeBuffers now */
 static int createBuffersFromMemory(buffers_t* buff, void * srcBuffer, size_t nbFiles,
     const size_t* fileSizes)
@@ -559,7 +694,7 @@ static int createBuffersFromMemory(buffers_t* buff, void * srcBuffer, size_t nbF
 
     if(!buff->srcPtrs || !buff->srcSizes || !buff->dstPtrs || !buff->dstCapacities || !buff->dstSizes || !buff->resPtrs || !buff->resSizes) {
         DISPLAY("alloc error\n");
-        freeBuffers(*buff);
+        freeNonSrcBuffers(*buff);
         return 1;
     }
 
@@ -570,7 +705,7 @@ static int createBuffersFromMemory(buffers_t* buff, void * srcBuffer, size_t nbF
 
     if(!buff->dstPtrs[0] || !buff->resPtrs[0]) {
         DISPLAY("alloc error\n");
-        freeBuffers(*buff);
+        freeNonSrcBuffers(*buff);
         return 1;
     }
 
@@ -611,12 +746,20 @@ static int createBuffers(buffers_t* buff, const char* const * const fileNamesTab
     size_t totalSizeToLoad = UTIL_getTotalFileSize(fileNamesTable, (U32)nbFiles);
     size_t benchedSize = MIN(BMK_findMaxMem(totalSizeToLoad * 3) / 3, totalSizeToLoad);
     size_t* fileSizes = calloc(sizeof(size_t), nbFiles);
-    void* srcBuffer = malloc(benchedSize);
+    void* srcBuffer = NULL; 
     int ret = 0;  
 
+    if(!totalSizeToLoad || !benchedSize) {
+        ret = 1;
+        DISPLAY("Nothing to Bench\n");
+        goto _cleanUp;
+    }
+
+    srcBuffer = malloc(benchedSize);
 
     if(!fileSizes || !srcBuffer) {
-        return 1;
+        ret = 1;
+        goto _cleanUp;
     }
 
     for(n = 0; n < nbFiles; n++) {
@@ -633,10 +776,9 @@ static int createBuffers(buffers_t* buff, const char* const * const fileNamesTab
         f = fopen(fileNamesTable[n], "rb");
         if (f==NULL) {
             DISPLAY("impossible to open file %s\n", fileNamesTable[n]);
-            free(fileSizes);
-            free(srcBuffer);
             fclose(f);
-            return 10;
+            ret = 10;
+            goto _cleanUp;
         }
 
         DISPLAY("Loading %s...       \r", fileNamesTable[n]);
@@ -645,21 +787,22 @@ static int createBuffers(buffers_t* buff, const char* const * const fileNamesTab
         {
             char* buffer = (char*)(srcBuffer); 
             size_t const readSize = fread((buffer)+pos, 1, (size_t)fileSize, f);
-
+            fclose(f);
             if (readSize != (size_t)fileSize) { /* should we accept partial read? */
                 DISPLAY("could not read %s", fileNamesTable[n]);
-                free(fileSizes);
-                free(srcBuffer);
-                return 1;
+                ret = 1;
+                goto _cleanUp;
             }
 
             fileSizes[n] = readSize;
             pos += readSize;
         }
-        fclose(f);
     }
 
     ret = createBuffersFromMemory(buff, srcBuffer, nbFiles, fileSizes);
+
+_cleanUp:
+    if(ret) { free(srcBuffer); }
     free(fileSizes);
     return ret;
 }
@@ -717,7 +860,7 @@ static int createContexts(contexts_t* ctx, const char* dictFileName) {
 /* if in decodeOnly, then srcPtr's will be compressed blocks, and uncompressedBlocks will be written to dstPtrs? */
 /* dictionary nullable, nothing else though. */
 static BMK_return_t BMK_benchMemInvertible(const buffers_t buf, const contexts_t ctx, 
-                        const int cLevel, const ZSTD_compressionParameters* comprParams,
+                        const int cLevel, const paramValues_t* comprParams,
                         const BMK_mode_t mode, const BMK_loopMode_t loopMode, const unsigned nbSeconds) {
 
     U32 i;
@@ -736,11 +879,6 @@ static BMK_return_t BMK_benchMemInvertible(const buffers_t buf, const contexts_t
     ZSTD_CCtx* cctx = ctx.cctx;
     ZSTD_DCtx* dctx = ctx.dctx;
 
-    BMK_advancedParams_t adv = BMK_initAdvancedParams();
-    adv.mode = mode;
-    adv.loopMode = loopMode;
-    adv.nbSeconds = nbSeconds;
-
     /* warmimg up memory */
     /* can't do this if decode only */
     for(i = 0; i < buf.nbBlocks; i++) {
@@ -762,7 +900,6 @@ static BMK_return_t BMK_benchMemInvertible(const buffers_t buf, const contexts_t
         cctxprep.dictBufferSize = dictBufferSize;
         cctxprep.cLevel = cLevel;
         cctxprep.comprParams = comprParams;
-        cctxprep.adv = &adv;
         dctxprep.dctx = dctx;
         dctxprep.dictBuffer = dictBuffer;
         dctxprep.dictBufferSize = dictBufferSize;
@@ -852,13 +989,13 @@ static BMK_return_t BMK_benchMemInvertible(const buffers_t buf, const contexts_t
         }
     }
    /* Bench */
-    results.result.cMem = (1 << (comprParams->windowLog)) + ZSTD_sizeof_CCtx(cctx);
+    results.result.cMem = (1 << (comprParams->vals[wlog_ind])) + ZSTD_sizeof_CCtx(cctx);
     return results;
 }
 
 static int BMK_benchParam(BMK_result_t* resultPtr,
                 buffers_t buf, contexts_t ctx,
-                const ZSTD_compressionParameters cParams) {
+                const paramValues_t cParams) {
     BMK_return_t res = BMK_benchMemInvertible(buf, ctx, BASE_CLEVEL, &cParams, BMK_both, BMK_timeMode, 3);
     *resultPtr = res.result;
     return res.error;
@@ -986,23 +1123,21 @@ static int insertWinner(winnerInfo_t w, constraint_t targetConstraints) {
 
 /* Writes to f the results of a parameter benchmark */
 /* when used with --optimize, will only print results better than previously discovered */
-static void BMK_printWinner(FILE* f, const U32 cLevel, const BMK_result_t result, const ZSTD_compressionParameters params, const size_t srcSize)
+static void BMK_printWinner(FILE* f, const U32 cLevel, const BMK_result_t result, const paramValues_t params, const size_t srcSize)
 {
     char lvlstr[15] = "Custom Level";
+    winnerInfo_t w;
+    w.params = params;
+    w.result = result;
 
     fprintf(f, "\r%79s\r", "");
 
-    fprintf(f,"    {%3u,%3u,%3u,%3u,%3u,%3u, %s },  ",
-        params.windowLog, params.chainLog, params.hashLog, params.searchLog, params.searchLength,
-        params.targetLength, g_stratName[(U32)(params.strategy)]);
-
     if(cLevel != CUSTOM_LEVEL) {
         snprintf(lvlstr, 15, "  Level %2u  ", cLevel);
     }
 
-    fprintf(f,
-        "/* %s */   /* R:%5.3f at %5.1f MB/s - %5.1f MB/s */",
-        lvlstr, (double)srcSize / result.cSize, (double)result.cSpeed / (1 MB), (double)result.dSpeed / (1 MB));
+    fprintf(f, "/* %s */   ", lvlstr);
+    BMK_displayOneResult(f, w, srcSize);
 
     if(TIMED) {
         const U64 time = UTIL_clockSpanNano(g_time);
@@ -1012,11 +1147,10 @@ static void BMK_printWinner(FILE* f, const U32 cLevel, const BMK_result_t result
     fprintf(f, "\n"); 
 }
 
-static void BMK_printWinnerOpt(FILE* f, const U32 cLevel, const BMK_result_t result, const ZSTD_compressionParameters params, const constraint_t targetConstraints, const size_t srcSize)
+static void BMK_printWinnerOpt(FILE* f, const U32 cLevel, const BMK_result_t result, const paramValues_t params, const constraint_t targetConstraints, const size_t srcSize)
 {
     /* global winner used for constraints */
-    static winnerInfo_t g_winner = { { 0, 0, (size_t)-1, (size_t)-1 } , { 0, 0, 0, 0, 0, 0, ZSTD_fast } }; 
-
+    static winnerInfo_t g_winner = { { (size_t)-1LL, 0, 0,  (size_t)-1LL }, { { PARAM_UNSET, PARAM_UNSET, PARAM_UNSET, PARAM_UNSET, PARAM_UNSET, PARAM_UNSET, PARAM_UNSET, PARAM_UNSET } } };
     if(DEBUG || compareResultLT(g_winner.result, result, targetConstraints, srcSize)) {
         if(DEBUG && compareResultLT(g_winner.result, result, targetConstraints, srcSize)) {
             DISPLAY("New Winner: \n");
@@ -1025,7 +1159,7 @@ static void BMK_printWinnerOpt(FILE* f, const U32 cLevel, const BMK_result_t res
         BMK_printWinner(f, cLevel, result, params, srcSize);
 
         if(compareResultLT(g_winner.result, result, targetConstraints, srcSize)) {
-            BMK_translateAdvancedParams(params);
+            BMK_translateAdvancedParams(f, params);
             g_winner.result = result;
             g_winner.params = params;
         }
@@ -1046,13 +1180,7 @@ static void BMK_printWinnerOpt(FILE* f, const U32 cLevel, const BMK_result_t res
         fprintf(f, "================================\n");
         for(n = g_winners; n != NULL; n = n->next) {
             fprintf(f, "\r%79s\r", "");
-
-            fprintf(f,"    {%3u,%3u,%3u,%3u,%3u,%3u, %s },  ",
-                n->res.params.windowLog, n->res.params.chainLog, n->res.params.hashLog, n->res.params.searchLog, n->res.params.searchLength,
-                n->res.params.targetLength, g_stratName[(U32)(n->res.params.strategy)]);
-            fprintf(f,
-            "   /* R:%5.3f at %5.1f MB/s - %5.1f MB/s */\n",
-            (double)srcSize / n->res.result.cSize, (double)n->res.result.cSpeed / (1 MB), (double)n->res.result.dSpeed / (1 MB));
+            BMK_displayOneResult(f, n->res, srcSize);
         }
         fprintf(f, "================================\n");
         fprintf(f, "Level Bounds: R: > %.3f AND C: < %.1f MB/s \n\n",
@@ -1060,27 +1188,15 @@ static void BMK_printWinnerOpt(FILE* f, const U32 cLevel, const BMK_result_t res
 
 
         fprintf(f, "Overall Winner: \n");
-        fprintf(f,"    {%3u,%3u,%3u,%3u,%3u,%3u, %s },  ",
-            g_winner.params.windowLog, g_winner.params.chainLog, g_winner.params.hashLog, g_winner.params.searchLog, g_winner.params.searchLength,
-            g_winner.params.targetLength, g_stratName[(U32)(g_winner.params.strategy)]);
-        fprintf(f,
-        "   /* R:%5.3f at %5.1f MB/s - %5.1f MB/s */\n",
-        (double)srcSize / g_winner.result.cSize, (double)g_winner.result.cSpeed / (1 MB), (double)g_winner.result.dSpeed / (1 MB));
-
-        BMK_translateAdvancedParams(g_winner.params);
-
-        fprintf(f, "Latest BMK: \n");
-        fprintf(f,"    {%3u,%3u,%3u,%3u,%3u,%3u, %s },  ",
-            params.windowLog, params.chainLog, params.hashLog, params.searchLog, params.searchLength,
-            params.targetLength, g_stratName[(U32)(params.strategy)]);
-        fprintf(f,
-            "   /* R:%5.3f at %5.1f MB/s - %5.1f MB/s */\n",
-            (double)srcSize / result.cSize, (double)result.cSpeed / (1 MB), (double)result.dSpeed / (1 MB));
+        BMK_displayOneResult(f, g_winner, srcSize);
+        BMK_translateAdvancedParams(f, g_winner.params);
 
+        fprintf(f, "Latest BMK: \n");\
+        BMK_displayOneResult(f, w, srcSize);
     }
 }
 
-static void BMK_printWinners2(FILE* f, const winnerInfo_t* winners, size_t srcSize)
+static void BMK_printWinners2(FILE* f, const oldWinnerInfo_t* winners, size_t srcSize)
 {
     int cLevel;
 
@@ -1088,11 +1204,11 @@ static void BMK_printWinners2(FILE* f, const winnerInfo_t* winners, size_t srcSi
     fprintf(f, "    /* W,  C,  H,  S,  L,  T, strat */ \n");
 
     for (cLevel=0; cLevel <= NB_LEVELS_TRACKED; cLevel++)
-        BMK_printWinner(f, cLevel, winners[cLevel].result, winners[cLevel].params, srcSize);
+        BMK_printWinner(f, cLevel, winners[cLevel].result, cParamsToPVals(winners[cLevel].params), srcSize);
 }
 
 
-static void BMK_printWinners(FILE* f, const winnerInfo_t* winners, size_t srcSize)
+static void BMK_printWinners(FILE* f, const oldWinnerInfo_t* winners, size_t srcSize)
 {
     fseek(f, 0, SEEK_SET);
     BMK_printWinners2(f, winners, srcSize);
@@ -1129,14 +1245,14 @@ static void BMK_init_level_constraints(int bytePerSec_level1)
     }   }
 }
 
-static int BMK_seed(winnerInfo_t* winners, const ZSTD_compressionParameters params, 
+static int BMK_seed(oldWinnerInfo_t* winners, const ZSTD_compressionParameters params, 
                     buffers_t buf, contexts_t ctx)
 {
     BMK_result_t testResult;
     int better = 0;
     int cLevel;
 
-    BMK_benchParam(&testResult, buf, ctx, params);
+    BMK_benchParam(&testResult, buf, ctx, cParamsToPVals(params));
 
 
     for (cLevel = 1; cLevel <= NB_LEVELS_TRACKED; cLevel++) {
@@ -1152,7 +1268,7 @@ static int BMK_seed(winnerInfo_t* winners, const ZSTD_compressionParameters para
             /* first solution for this cLevel */
             winners[cLevel].result = testResult;
             winners[cLevel].params = params;
-            BMK_printWinner(stdout, cLevel, testResult, params, buf.srcSize);
+            BMK_printWinner(stdout, cLevel, testResult, cParamsToPVals(params), buf.srcSize);
             better = 1;
             continue;
         }
@@ -1217,7 +1333,7 @@ static int BMK_seed(winnerInfo_t* winners, const ZSTD_compressionParameters para
 
             winners[cLevel].result = testResult;
             winners[cLevel].params = params;
-            BMK_printWinner(stdout, cLevel, testResult, params, buf.srcSize);
+            BMK_printWinner(stdout, cLevel, testResult, cParamsToPVals(params), buf.srcSize);
 
             better = 1;
     }   }
@@ -1234,20 +1350,21 @@ static int BMK_seed(winnerInfo_t* winners, const ZSTD_compressionParameters para
 /* no point in windowLog < chainLog (no point 2x chainLog for bt) */
 /* now with built in bounds-checking */
 /* no longer does anything with sanitizeVarArray + clampcheck */
-static ZSTD_compressionParameters sanitizeParams(ZSTD_compressionParameters params)
+static paramValues_t sanitizeParams(paramValues_t params)
 {
-    if (params.strategy == ZSTD_fast)
-        params.chainLog = 0, params.searchLog = 0;
-    if (params.strategy == ZSTD_dfast)
-        params.searchLog = 0;
-    if (params.strategy != ZSTD_btopt && params.strategy != ZSTD_btultra && params.strategy != ZSTD_fast)
-        params.targetLength = 0;
+    if (params.vals[strt_ind] == ZSTD_fast)
+        params.vals[clog_ind] = 0, params.vals[slog_ind] = 0;
+    if (params.vals[strt_ind] == ZSTD_dfast)
+        params.vals[slog_ind] = 0;
+    if (params.vals[strt_ind] != ZSTD_btopt && params.vals[strt_ind] != ZSTD_btultra && params.vals[strt_ind] != ZSTD_fast)
+        params.vals[tlen_ind] = 0;
 
     return params;
 }
 
-/* new length */
+/* return: new length */
 /* keep old array, will need if iter over strategy. */
+/* prunes useless params */
 static int sanitizeVarArray(varInds_t* varNew, const int varLength, const varInds_t* varArray, const ZSTD_strategy strat) {
     int i, j = 0;
     for(i = 0; i < varLength; i++) {
@@ -1264,80 +1381,31 @@ static int sanitizeVarArray(varInds_t* varNew, const int varLength, const varInd
 }
 
 /* res should be NUM_PARAMS size */
-/* constructs varArray from ZSTD_compressionParameters style parameter */
-static int variableParams(const ZSTD_compressionParameters paramConstraints, varInds_t* res) {
+/* constructs varArray from paramValues_t style parameter */
+/* pass in using dict. */
+static int variableParams(const paramValues_t paramConstraints, varInds_t* res, const int usingDictionary) {
+    varInds_t i;
     int j = 0;
-    if(!paramConstraints.windowLog) {
-        res[j] = wlog_ind;
-        j++;
-    }
-    if(!paramConstraints.chainLog) {
-        res[j] = clog_ind;
-        j++;
-    }
-    if(!paramConstraints.hashLog) {
-        res[j] = hlog_ind;
-        j++;
-    }
-    if(!paramConstraints.searchLog) {
-        res[j] = slog_ind;
-        j++;
-    }
-    if(!paramConstraints.searchLength) {
-        res[j] = slen_ind;
-        j++;
-    }
-    if(!paramConstraints.targetLength) {
-        res[j] = tlen_ind;
-        j++;
-    }
-    if(!paramConstraints.strategy) {
-        res[j] = strt_ind;
-        j++;
-    }
-    return j;
-}
-
-/* bin-search on tlen_table for correct index. */
-static int tlen_inv(U32 x) {
-    int lo = 0;
-    int hi = TLEN_RANGE;
-    while(lo < hi) {
-        int mid = (lo + hi) / 2;
-        if(tlen_table[mid] < x) {
-            lo = mid + 1;
-        } if(tlen_table[mid] == x) {
-            return mid;
-        } else {
-            hi = mid;
+    for(i = 0; i < NUM_PARAMS; i++) {
+        if(paramConstraints.vals[i] == PARAM_UNSET) {
+            if(i == fadt_ind && !usingDictionary) continue; /* don't use fadt if no dictionary */
+            res[j] = i; j++;
         }
     }
-    return lo;
+    return j;
 }
 
 /* amt will probably always be \pm 1? */
 /* slight change from old paramVariation, targetLength can only take on powers of 2 now (999 ~= 1024?) */
 /* take max/min bounds into account as well? */
-static void paramVaryOnce(const varInds_t paramIndex, const int amt, ZSTD_compressionParameters* ptr) {
-    switch(paramIndex)
-    {
-        case wlog_ind: ptr->windowLog    += amt; break;
-        case clog_ind: ptr->chainLog     += amt; break;
-        case hlog_ind: ptr->hashLog      += amt; break;
-        case slog_ind: ptr->searchLog    += amt; break;
-        case slen_ind: ptr->searchLength += amt; break;
-        case tlen_ind: 
-            ptr->targetLength = tlen_table[MAX(0, MIN(TLEN_RANGE - 1, tlen_inv(ptr->targetLength) + amt))];
-            break;
-        case strt_ind: ptr->strategy     += amt; break;
-        default: break;
-    }
+static void paramVaryOnce(const varInds_t paramIndex, const int amt, paramValues_t* ptr) {
+    ptr->vals[paramIndex] = rangeMap(paramIndex, invRangeMap(paramIndex, ptr->vals[paramIndex]) + amt); //TODO: bounds check. 
 }
 
 /* varies ptr by nbChanges respecting varyParams*/
-static void paramVariation(ZSTD_compressionParameters* ptr, const varInds_t* varyParams, const int varyLen, const U32 nbChanges)
+static void paramVariation(paramValues_t* ptr, const varInds_t* varyParams, const int varyLen, const U32 nbChanges)
 {
-    ZSTD_compressionParameters p;
+    paramValues_t p;
     U32 validated = 0;
     while (!validated) {
         U32 i;
@@ -1346,7 +1414,7 @@ static void paramVariation(ZSTD_compressionParameters* ptr, const varInds_t* var
             const U32 changeID = FUZ_rand(&g_rand) % (varyLen << 1);
             paramVaryOnce(varyParams[changeID >> 1], ((changeID & 1) << 1) - 1, &p);
         }
-        validated = !ZSTD_isError(ZSTD_checkCParams(p)) && p.strategy > 0;
+        validated = paramValid(p);
     }
     *ptr = p;
 }
@@ -1364,48 +1432,25 @@ static size_t memoTableLen(const varInds_t* varyParams, const int varyLen) {
 }
 
 /* returns unique index in memotable of compression parameters */
-static unsigned memoTableInd(const ZSTD_compressionParameters* ptr, const varInds_t* varyParams, const int varyLen) {
+static unsigned memoTableInd(const paramValues_t* ptr, const varInds_t* varyParams, const int varyLen) {
     int i;
     unsigned ind = 0;
     for(i = 0; i < varyLen; i++) {
-        switch(varyParams[i]) {
-            case wlog_ind: ind *= WLOG_RANGE; ind += ptr->windowLog              
-                - ZSTD_WINDOWLOG_MIN   ; break;
-            case clog_ind: ind *= CLOG_RANGE; ind += ptr->chainLog               
-                - ZSTD_CHAINLOG_MIN    ; break;
-            case hlog_ind: ind *= HLOG_RANGE; ind += ptr->hashLog                
-                - ZSTD_HASHLOG_MIN     ; break;
-            case slog_ind: ind *= SLOG_RANGE; ind += ptr->searchLog              
-                - ZSTD_SEARCHLOG_MIN   ; break;
-            case slen_ind: ind *= SLEN_RANGE; ind += ptr->searchLength           
-                - ZSTD_SEARCHLENGTH_MIN; break;
-            case tlen_ind: ind *= TLEN_RANGE; ind += tlen_inv(ptr->targetLength) 
-                - ZSTD_TARGETLENGTH_MIN; break;
-            case strt_ind: break;
-        }
+        varInds_t v = varyParams[i];
+        if(v == strt_ind) continue; /* exclude strategy from memotable */
+        ind *= rangetable[v]; ind += invRangeMap(v, ptr->vals[v]);
     }
     return ind;
 }
 
 /* inverse of above function (from index to parameters) */
-static void memoTableIndInv(ZSTD_compressionParameters* ptr, const varInds_t* varyParams, const int varyLen, size_t ind) {
+static void memoTableIndInv(paramValues_t* ptr, const varInds_t* varyParams, const int varyLen, size_t ind) {
     int i;
     for(i = varyLen - 1; i >= 0; i--) {
-        switch(varyParams[i]) {
-            case wlog_ind: ptr->windowLog    = ind % WLOG_RANGE + ZSTD_WINDOWLOG_MIN;
-                ind /= WLOG_RANGE; break;
-            case clog_ind: ptr->chainLog     = ind % CLOG_RANGE + ZSTD_CHAINLOG_MIN;
-                ind /= CLOG_RANGE; break;
-            case hlog_ind: ptr->hashLog      = ind % HLOG_RANGE + ZSTD_HASHLOG_MIN;
-                ind /= HLOG_RANGE; break;
-            case slog_ind: ptr->searchLog    = ind % SLOG_RANGE + ZSTD_SEARCHLOG_MIN;
-                ind /= SLOG_RANGE; break;
-            case slen_ind: ptr->searchLength = ind % SLEN_RANGE + ZSTD_SEARCHLENGTH_MIN;
-                ind /= SLEN_RANGE; break;
-            case tlen_ind: ptr->targetLength = tlen_table[(ind % TLEN_RANGE)];
-                ind /= TLEN_RANGE; break;
-            case strt_ind: break;
-        }
+        varInds_t v = varyParams[i];
+        if(v == strt_ind) continue;
+        ptr->vals[v] = rangeMap(v, ind % rangetable[v]);
+        ind /= rangetable[v];
     }
 }
 
@@ -1414,36 +1459,36 @@ static void memoTableIndInv(ZSTD_compressionParameters* ptr, const varInds_t* va
  * redundant / obviously non-optimal parameter configurations (e.g. wlog - 1 larger)
  * than srcSize, clog > wlog, ... 
  */
-static void initMemoTable(U8* memoTable, ZSTD_compressionParameters paramConstraints, const constraint_t target, const varInds_t* varyParams, const int varyLen, const size_t srcSize) {
+static void initMemoTable(U8* memoTable, paramValues_t paramConstraints, const constraint_t target, const varInds_t* varyParams, const int varyLen, const size_t srcSize) {
     size_t i;
     size_t arrayLen = memoTableLen(varyParams, varyLen);
-    int cwFixed = !paramConstraints.chainLog || !paramConstraints.windowLog;
-    int scFixed = !paramConstraints.searchLog || !paramConstraints.chainLog;
-    int whFixed = !paramConstraints.windowLog || !paramConstraints.hashLog;
-    int wFixed = !paramConstraints.windowLog;
+    int cwFixed = paramConstraints.vals[clog_ind] == PARAM_UNSET || paramConstraints.vals[wlog_ind] == PARAM_UNSET;
+    int scFixed = paramConstraints.vals[slog_ind] == PARAM_UNSET || paramConstraints.vals[clog_ind] == PARAM_UNSET;
+    int whFixed = paramConstraints.vals[wlog_ind] == PARAM_UNSET || paramConstraints.vals[hlog_ind] == PARAM_UNSET;
+    int wFixed = paramConstraints.vals[wlog_ind] == PARAM_UNSET;
     int j = 0;
     assert(memoTable != NULL);
     memset(memoTable, 0, arrayLen);
-    cParamZeroMin(¶mConstraints);
+    paramConstraints = cParamUnsetMin(paramConstraints);
 
     for(i = 0; i < arrayLen; i++) {
         memoTableIndInv(¶mConstraints, varyParams, varyLen, i);
-        if(ZSTD_estimateCStreamSize_usingCParams(paramConstraints) > (size_t)target.cMem) {
+        if(ZSTD_estimateCStreamSize_usingCParams(pvalsToCParams(paramConstraints)) > (size_t)target.cMem) {
             memoTable[i] = 255;
             j++;
         }
-        if(wFixed && (1ULL << (paramConstraints.windowLog - 1)) > srcSize) {
+        if(wFixed && (1ULL << (paramConstraints.vals[wlog_ind] - 1)) >= srcSize && paramConstraints.vals[wlog_ind] != mintable[wlog_ind]) {
             memoTable[i] = 255;
         }
         /* nil out parameter sets equivalent to others. */
-        if(cwFixed/* at most least 1 param fixed. */) {
-            if(paramConstraints.strategy == ZSTD_btlazy2 || paramConstraints.strategy == ZSTD_btopt || paramConstraints.strategy == ZSTD_btultra) {
-                if(paramConstraints.chainLog > paramConstraints.windowLog + 1) {
+        if(cwFixed) {
+            if(paramConstraints.vals[strt_ind] == ZSTD_btlazy2 || paramConstraints.vals[strt_ind] == ZSTD_btopt || paramConstraints.vals[strt_ind] == ZSTD_btultra) {
+                if(paramConstraints.vals[clog_ind] > paramConstraints.vals[wlog_ind]+ 1) {
                     if(memoTable[i] != 255) { j++; }
                     memoTable[i] = 255;
                 }
             } else {
-                if(paramConstraints.chainLog > paramConstraints.windowLog) {
+                if(paramConstraints.vals[clog_ind] > paramConstraints.vals[wlog_ind]) {
                     if(memoTable[i] != 255) { j++; }
                     memoTable[i] = 255;
                 }
@@ -1451,14 +1496,14 @@ static void initMemoTable(U8* memoTable, ZSTD_compressionParameters paramConstra
         }
 
         if(scFixed) {
-            if(paramConstraints.searchLog > paramConstraints.chainLog) {
+            if(paramConstraints.vals[slog_ind] > paramConstraints.vals[clog_ind]) {
                 if(memoTable[i] != 255) { j++; }
                 memoTable[i] = 255;
             }
         }
 
         if(whFixed) {
-            if(paramConstraints.hashLog > paramConstraints.windowLog + 1) {
+            if(paramConstraints.vals[hlog_ind] > paramConstraints.vals[wlog_ind] + 1) {
                 if(memoTable[i] != 255) { j++; }
                 memoTable[i] = 255;
             }
@@ -1466,7 +1511,7 @@ static void initMemoTable(U8* memoTable, ZSTD_compressionParameters paramConstra
     }
     DEBUGOUTPUT("%d / %d Invalid\n", j, (int)i);
     if((int)i == j) {
-        DEBUGOUTPUT("!!!Strategy %d totally infeasible\n", (int)paramConstraints.strategy)
+        DEBUGOUTPUT("!!!Strategy %d totally infeasible\n", (int)paramConstraints.vals[strt_ind]);
     }
 }
 
@@ -1482,7 +1527,7 @@ static void freeMemoTableArray(U8** mtAll) {
 
 /* inits memotables for all (including mallocs), all strategies */
 /* takes unsanitized varyParams */
-static U8** createMemoTableArray(ZSTD_compressionParameters paramConstraints, constraint_t target, const varInds_t* varyParams, const int varyLen, const size_t srcSize) {
+static U8** createMemoTableArray(paramValues_t paramConstraints, constraint_t target, const varInds_t* varyParams, const int varyLen, const size_t srcSize) {
     varInds_t varNew[NUM_PARAMS];
     U8** mtAll = (U8**)calloc(sizeof(U8*),(ZSTD_btultra + 1));
     int i;
@@ -1503,14 +1548,13 @@ static U8** createMemoTableArray(ZSTD_compressionParameters paramConstraints, co
     return mtAll;
 }
 
-static ZSTD_compressionParameters overwriteParams(ZSTD_compressionParameters base, ZSTD_compressionParameters mask) {
-    base.windowLog = mask.windowLog ? mask.windowLog : base.windowLog;
-    base.chainLog = mask.chainLog ? mask.chainLog : base.chainLog;
-    base.hashLog = mask.hashLog ? mask.hashLog : base.hashLog;
-    base.searchLog = mask.searchLog ? mask.searchLog : base.searchLog;
-    base.searchLength = mask.searchLength ? mask.searchLength : base.searchLength;
-    base.targetLength = mask.targetLength ? mask.targetLength : base.targetLength;
-    base.strategy = mask.strategy ? mask.strategy : base.strategy;
+static paramValues_t overwriteParams(paramValues_t base, paramValues_t mask) {
+    U32 i;
+    for(i = 0; i < NUM_PARAMS; i++) {
+        if(mask.vals[i] != PARAM_UNSET) {
+            base.vals[i] = mask.vals[i];
+        }
+    }
     return base;
 }
 
@@ -1524,38 +1568,41 @@ static BYTE g_alreadyTested[PARAMTABLESIZE] = {0};   /* init to zero */
     g_alreadyTested[(XXH64(((void*)&sanitizeParams(p), sizeof(p), 0) >> 3) & PARAMTABLEMASK] */
 
 static BYTE* NB_TESTS_PLAYED(ZSTD_compressionParameters p) {
-    ZSTD_compressionParameters p2 = sanitizeParams(p);
+    ZSTD_compressionParameters p2 = pvalsToCParams(sanitizeParams(cParamsToPVals(p)));
     return &g_alreadyTested[(XXH64((void*)&p2, sizeof(p2), 0) >> 3) & PARAMTABLEMASK];
 }
 
-static void playAround(FILE* f, winnerInfo_t* winners,
+static void playAround(FILE* f, oldWinnerInfo_t* winners,
                        ZSTD_compressionParameters params,
                        buffers_t buf, contexts_t ctx)
 {
     int nbVariations = 0;
     UTIL_time_t const clockStart = UTIL_getTime();
-    const U32 unconstrained[NUM_PARAMS] = { 0, 1, 2, 3, 4, 5, 6 };
+    const U32 unconstrained[NUM_PARAMS] = { 0, 1, 2, 3, 4, 5, 6 }; /* no fadt */
 
 
     while (UTIL_clockSpanMicro(clockStart) < g_maxVariationTime) {
-        ZSTD_compressionParameters p = params;
+        paramValues_t p = cParamsToPVals(params);
+        ZSTD_compressionParameters p2;
         BYTE* b;
 
         if (nbVariations++ > g_maxNbVariations) break;
         paramVariation(&p, unconstrained, NUM_PARAMS, 4);
 
+        p2 = pvalsToCParams(p);
+
         /* exclude faster if already played params */
-        if (FUZ_rand(&g_rand) & ((1 << *NB_TESTS_PLAYED(p))-1))
+        if (FUZ_rand(&g_rand) & ((1 << *NB_TESTS_PLAYED(p2))-1))
             continue;
 
         /* test */
-        b = NB_TESTS_PLAYED(p);
+        b = NB_TESTS_PLAYED(p2);
         (*b)++;
-        if (!BMK_seed(winners, p, buf, ctx)) continue;
+        if (!BMK_seed(winners, p2, buf, ctx)) continue;
 
         /* improvement found => search more */
         BMK_printWinners(f, winners, buf.srcSize);
-        playAround(f, winners, p, buf, ctx);
+        playAround(f, winners, p2, buf, ctx);
     }
 
 }
@@ -1587,7 +1634,7 @@ static ZSTD_compressionParameters randomParams(void)
 }
 
 /* Sets pc to random unmeasured set of parameters */
-static void randomConstrainedParams(ZSTD_compressionParameters* pc, varInds_t* varArray, int varLen, U8* memoTable)
+static void randomConstrainedParams(paramValues_t* pc, varInds_t* varArray, int varLen, U8* memoTable)
 {
     size_t tries = memoTableLen(varArray, varLen); 
     const size_t maxSize = memoTableLen(varArray, varLen);
@@ -1601,7 +1648,7 @@ static void randomConstrainedParams(ZSTD_compressionParameters* pc, varInds_t* v
 }
 
 static void BMK_selectRandomStart(
-                       FILE* f, winnerInfo_t* winners,
+                       FILE* f, oldWinnerInfo_t* winners,
                        buffers_t buf, contexts_t ctx)
 {
     U32 const id = FUZ_rand(&g_rand) % (NB_LEVELS_TRACKED+1);
@@ -1617,7 +1664,7 @@ static void BMK_selectRandomStart(
 static void BMK_benchFullTable(buffers_t buf, contexts_t ctx, const size_t maxBlockSize) 
 {
     ZSTD_compressionParameters params;
-    winnerInfo_t winners[NB_LEVELS_TRACKED+1];
+    oldWinnerInfo_t winners[NB_LEVELS_TRACKED+1];
     const char* const rfName = "grillResults.txt";
     FILE* const f = fopen(rfName, "w");
 
@@ -1632,7 +1679,7 @@ static void BMK_benchFullTable(buffers_t buf, contexts_t ctx, const size_t maxBl
         /* baseline config for level 1 */
         ZSTD_compressionParameters const l1params = ZSTD_getCParams(1, maxBlockSize, ctx.dictSize);
         BMK_result_t testResult;
-        BMK_benchParam(&testResult, buf, ctx, l1params);
+        BMK_benchParam(&testResult, buf, ctx, cParamsToPVals(l1params));
         BMK_init_level_constraints((int)((testResult.cSpeed * 31) / 32));
     }
 
@@ -1751,7 +1798,7 @@ int benchFiles(const char** fileNamesTable, int nbFiles, const char* dictFileNam
         DISPLAY("using %d Files : \n", nbFiles);
     }
 
-    g_params = ZSTD_adjustCParams(overwriteParams(ZSTD_getCParams(cLevel, maxBlockSize, ctx.dictSize), g_params), maxBlockSize, ctx.dictSize);
+    g_params = adjustParams(overwriteParams(cParamsToPVals(ZSTD_getCParams(cLevel, maxBlockSize, ctx.dictSize)), g_params), maxBlockSize, ctx.dictSize);
 
     if(g_singleRun) {
         ret = benchOnce(buf, ctx);
@@ -1769,7 +1816,7 @@ int benchFiles(const char** fileNamesTable, int nbFiles, const char* dictFileNam
 #define HIGH_VARIANCE 100.0
 static int allBench(BMK_result_t* resultPtr,
                 const buffers_t buf, const contexts_t ctx,
-                const ZSTD_compressionParameters cParams,
+                const paramValues_t cParams,
                 const constraint_t target,
                 BMK_result_t* winnerResult, int feas) {
     BMK_return_t benchres;
@@ -1870,7 +1917,7 @@ static int allBench(BMK_result_t* resultPtr,
 /* Memoized benchmarking, won't benchmark anything which has already been benchmarked before. */
 static int benchMemo(BMK_result_t* resultPtr,
                 const buffers_t buf, const contexts_t ctx, 
-                const ZSTD_compressionParameters cParams,
+                const paramValues_t cParams,
                 const constraint_t target,
                 BMK_result_t* winnerResult, U8* const memoTable,
                 const varInds_t* varyParams, const int varyLen, const int feas) {
@@ -1914,13 +1961,13 @@ static winnerInfo_t climbOnce(const constraint_t target,
                 const varInds_t* varArray, const int varLen, ZSTD_strategy strat,
                 U8** memoTableArray, 
                 buffers_t buf, contexts_t ctx,
-                const ZSTD_compressionParameters init) {
+                const paramValues_t init) {
     /* 
      * cparam - currently considered 'center'
      * candidate - params to benchmark/results
      * winner - best option found so far.
      */
-    ZSTD_compressionParameters cparam = init;
+    paramValues_t cparam = init;
     winnerInfo_t candidateInfo, winnerInfo;
     int better = 1;
     int feas = 0;
@@ -1948,10 +1995,10 @@ static winnerInfo_t climbOnce(const constraint_t target,
                     candidateInfo.params = cparam;
                     paramVaryOnce(varArray[i], offset, &candidateInfo.params); 
                     
-                    if(!ZSTD_isError(ZSTD_checkCParams(candidateInfo.params)) && candidateInfo.params.strategy > 0) {
+                    if(paramValid(candidateInfo.params)) {
                         int res;
-                        if(strat != candidateInfo.params.strategy) { /* maybe only try strategy switching after exhausting non-switching solutions? */
-                            strat = candidateInfo.params.strategy;
+                        if(strat != candidateInfo.params.vals[strt_ind]) { /* maybe only try strategy switching after exhausting non-switching solutions? */
+                            strat = candidateInfo.params.vals[strt_ind];
                             varLenNew = sanitizeVarArray(varNew, varLen, varArray, strat);
                         }
                         res = benchMemo(&candidateInfo.result, buf, ctx,
@@ -1981,8 +2028,8 @@ static winnerInfo_t climbOnce(const constraint_t target,
                     /* param error checking already done here */
                     paramVariation(&candidateInfo.params, varArray, varLen, dist);
 
-                    if(strat != candidateInfo.params.strategy) {
-                        strat = candidateInfo.params.strategy;
+                    if(strat != candidateInfo.params.vals[strt_ind]) {
+                        strat = candidateInfo.params.vals[strt_ind];
                         varLenNew = sanitizeVarArray(varNew, varLen, varArray, strat);
                     }
 
@@ -2030,7 +2077,7 @@ static winnerInfo_t climbOnce(const constraint_t target,
  */
 static winnerInfo_t optimizeFixedStrategy(
     const buffers_t buf, const contexts_t ctx, 
-    const constraint_t target, ZSTD_compressionParameters paramTarget,
+    const constraint_t target, paramValues_t paramTarget,
     const ZSTD_strategy strat, 
     const varInds_t* varArray, const int varLen,
     U8** memoTableArray, const int tries) {
@@ -2038,14 +2085,14 @@ static winnerInfo_t optimizeFixedStrategy(
     varInds_t varNew[NUM_PARAMS];
     int varLenNew = sanitizeVarArray(varNew, varLen, varArray, strat);
 
-    ZSTD_compressionParameters init;
+    paramValues_t init;
     winnerInfo_t winnerInfo, candidateInfo; 
     winnerInfo = initWinnerInfo(emptyParams());
     /* so climb is given the right fixed strategy */
-    paramTarget.strategy = strat;
+    paramTarget.vals[strt_ind] = strat;
     /* to pass ZSTD_checkCParams */
 
-    cParamZeroMin(¶mTarget);
+    paramTarget = cParamUnsetMin(paramTarget);
 
     init = paramTarget;
 
@@ -2112,25 +2159,23 @@ static int nextStrategy(const int currentStrategy, const int bestStrategy) {
 static int g_maxTries = 3;
 #define TRY_DECAY 1
 
-static int optimizeForSize(const char* const * const fileNamesTable, const size_t nbFiles, const char* dictFileName, constraint_t target, ZSTD_compressionParameters paramTarget, int cLevel)
+static int optimizeForSize(const char* const * const fileNamesTable, const size_t nbFiles, const char* dictFileName, constraint_t target, paramValues_t paramTarget, int cLevelOpt, int cLevelRun)
 {
     varInds_t varArray [NUM_PARAMS];
     int ret = 0;
-    const int varLen = variableParams(paramTarget, varArray);
+    const int varLen = variableParams(paramTarget, varArray, dictFileName != NULL);
     winnerInfo_t winner = initWinnerInfo(emptyParams());
     U8** allMT = NULL;
-    size_t k;
-    size_t maxBlockSize = 0;
+    paramValues_t paramBase = cParamUnsetMin(paramTarget);
+    size_t k, maxBlockSize = 0;
     contexts_t ctx;
     buffers_t buf;
     g_time = UTIL_getTime();
 
-    /* Init */
-    if(!cParamValid(paramTarget)) {
+    if(!paramValid(paramBase)) {
         return 1;
     }
 
-    /* load dictionary*/
     if(createBuffers(&buf, fileNamesTable, nbFiles)) {
         DISPLAY("unable to load files\n");
         return 1;
@@ -2154,23 +2199,23 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_
     }
 
     /* if strategy is fixed, only init that part of memotable */
-    if(paramTarget.strategy) {
+    if(paramTarget.vals[strt_ind] != PARAM_UNSET) {
         varInds_t varNew[NUM_PARAMS];
-        int varLenNew = sanitizeVarArray(varNew, varLen, varArray, paramTarget.strategy);
+        int varLenNew = sanitizeVarArray(varNew, varLen, varArray, paramTarget.vals[strt_ind]);
         allMT = (U8**)calloc(sizeof(U8*), (ZSTD_btultra + 1));
         if(allMT == NULL) {
             ret = 57;
             goto _cleanUp;
         }
 
-        allMT[paramTarget.strategy] = malloc(sizeof(U8) * memoTableLen(varNew, varLenNew));
+        allMT[paramTarget.vals[strt_ind]] = malloc(sizeof(U8) * memoTableLen(varNew, varLenNew));
         
-        if(allMT[paramTarget.strategy] == NULL) {
+        if(allMT[paramTarget.vals[strt_ind]] == NULL) {
             ret = 58;
             goto _cleanUp;
         }
         
-        initMemoTable(allMT[paramTarget.strategy], paramTarget, target, varNew, varLenNew, maxBlockSize);
+        initMemoTable(allMT[paramTarget.vals[strt_ind]], paramTarget, target, varNew, varLenNew, maxBlockSize);
     } else {
          allMT = createMemoTableArray(paramTarget, target, varArray, varLen, maxBlockSize);
     }
@@ -2180,7 +2225,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_
         ret = 2;
         goto _cleanUp;
     }
- 
+
     /* default strictness = Maximum for */
     if(g_strictness == DEFAULT_STRICTNESS) {
         if(g_optmode) {
@@ -2199,7 +2244,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_
     /* use level'ing mode instead of normal target mode */
     /* Should lvl be parameter-masked here? */
     if(g_optmode) {
-        winner.params = ZSTD_getCParams(cLevel, maxBlockSize, ctx.dictSize);
+        winner.params = cParamsToPVals(ZSTD_getCParams(cLevelOpt, maxBlockSize, ctx.dictSize));
         if(BMK_benchParam(&winner.result, buf, ctx, winner.params)) {
             ret = 3;
             goto _cleanUp;
@@ -2213,13 +2258,13 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_
         target.cSpeed = (U32)g_lvltarget.cSpeed;  
         target.dSpeed = (U32)g_lvltarget.dSpeed; //See if this is reasonable. 
 
-        BMK_printWinnerOpt(stdout, cLevel, winner.result, winner.params, target, buf.srcSize);
+        BMK_printWinnerOpt(stdout, cLevelOpt, winner.result, winner.params, target, buf.srcSize);
     }
 
     /* Don't want it to return anything worse than the best known result */
     if(g_singleRun) {
         BMK_result_t res;
-        g_params = ZSTD_adjustCParams(maskParams(ZSTD_getCParams(cLevel, maxBlockSize, ctx.dictSize), g_params), maxBlockSize, ctx.dictSize);
+        g_params = adjustParams(overwriteParams(cParamsToPVals(ZSTD_getCParams(cLevelRun, maxBlockSize, ctx.dictSize)), g_params), maxBlockSize, ctx.dictSize);
         if(BMK_benchParam(&res, buf, ctx, g_params)) {
             ret = 45;
             goto _cleanUp;
@@ -2246,19 +2291,19 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_
     findClockGranularity();
 
     {   
-        ZSTD_compressionParameters CParams;
+        paramValues_t CParams;
 
         /* find best solution from default params */
         {
             /* strategy selection */
             const int maxSeeds = g_noSeed ? 1 : ZSTD_maxCLevel();
             DEBUGOUTPUT("Strategy Selection\n");
-            if(paramTarget.strategy == 0) {
+            if(paramTarget.vals[strt_ind] == PARAM_UNSET) {
                 BMK_result_t candidate;
                 int i;
                 for (i=1; i<=maxSeeds; i++) {
                     int ec;
-                    CParams = overwriteParams(ZSTD_getCParams(i, maxBlockSize, ctx.dictSize), paramTarget);
+                    CParams = overwriteParams(cParamsToPVals(ZSTD_getCParams(i, maxBlockSize, ctx.dictSize)), paramTarget);
                     ec = BMK_benchParam(&candidate, buf, ctx, CParams);
                     BMK_printWinnerOpt(stdout, i, candidate, CParams, target, buf.srcSize);
 
@@ -2271,17 +2316,18 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_
                     /* if the current params are too slow, just stop. */
                     if(target.cSpeed > candidate.cSpeed * 3 / 2) { break; }
                 }
+
+                BMK_printWinnerOpt(stdout, CUSTOM_LEVEL, winner.result, winner.params, target, buf.srcSize);
+                BMK_translateAdvancedParams(stdout, winner.params);
             }
         }
 
-        BMK_printWinnerOpt(stdout, CUSTOM_LEVEL, winner.result, winner.params, target, buf.srcSize);
-        BMK_translateAdvancedParams(winner.params);
         DEBUGOUTPUT("Real Opt\n");
         /* start 'real' tests */
         {   
-            int bestStrategy = (int)winner.params.strategy;
-            if(paramTarget.strategy == 0) {
-                int st = (int)winner.params.strategy;
+            int bestStrategy = (int)winner.params.vals[strt_ind];
+            if(paramTarget.vals[strt_ind] == PARAM_UNSET) {
+                int st = bestStrategy;
                 int tries = g_maxTries;
 
                 { 
@@ -2298,7 +2344,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_
                     winnerInfo_t wc;
                     DEBUGOUTPUT("StrategySwitch: %s\n", g_stratName[st]);
                     
-                    wc = optimizeFixedStrategy(buf, ctx, target, paramTarget, 
+                    wc = optimizeFixedStrategy(buf, ctx, target, paramBase, 
                         st, varArray, varLen, allMT, tries);
 
                     if(compareResultLT(winner.result, wc.result, target, buf.srcSize)) {
@@ -2312,7 +2358,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_
                     CHECKTIMEGT(ret, 0, _cleanUp);
                 }
             } else {
-                winner = optimizeFixedStrategy(buf, ctx, target, paramTarget, paramTarget.strategy, 
+                winner = optimizeFixedStrategy(buf, ctx, target, paramBase, paramTarget.vals[strt_ind], 
                     varArray, varLen, allMT, g_maxTries);
             }
 
@@ -2326,7 +2372,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_
         }
         /* end summary */
         BMK_printWinnerOpt(stdout, CUSTOM_LEVEL, winner.result, winner.params, target, buf.srcSize);
-        BMK_translateAdvancedParams(winner.params);
+        BMK_translateAdvancedParams(stdout, winner.params);
         DISPLAY("grillParams size - optimizer completed \n");
 
     }
@@ -2350,7 +2396,9 @@ static void errorOut(const char* msg)
 static unsigned readU32FromChar(const char** stringPtr)
 {
     const char errorMsg[] = "error: numeric value too large";
+    unsigned sign = 1;
     unsigned result = 0;
+    if(**stringPtr == '-') { sign = (unsigned)-1; (*stringPtr)++; }
     while ((**stringPtr >='0') && (**stringPtr <='9')) {
         unsigned const max = (((unsigned)(-1)) / 10) - 1;
         if (result > max) errorOut(errorMsg);
@@ -2368,7 +2416,7 @@ static unsigned readU32FromChar(const char** stringPtr)
         if (**stringPtr=='i') (*stringPtr)++;
         if (**stringPtr=='B') (*stringPtr)++;
     }
-    return result;
+    return result * sign;
 }
 
 static int usage(const char* exename)
@@ -2408,13 +2456,14 @@ static int badusage(const char* exename)
 #define PARSE_SUB_ARGS(stringLong, stringShort, variable) { if (longCommandWArg(&argument, stringLong) || longCommandWArg(&argument, stringShort)) { variable = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; } }
 #define PARSE_CPARAMS(variable)                                            \
 {                                                                          \
-    PARSE_SUB_ARGS("windowLog=",       "wlog=",  variable.windowLog);      \
-    PARSE_SUB_ARGS("chainLog=" ,       "clog=",  variable.chainLog);       \
-    PARSE_SUB_ARGS("hashLog=",         "hlog=",  variable.hashLog);        \
-    PARSE_SUB_ARGS("searchLog=" ,      "slog=",  variable.searchLog);      \
-    PARSE_SUB_ARGS("searchLength=",    "slen=",  variable.searchLength);   \
-    PARSE_SUB_ARGS("targetLength=" ,   "tlen=",  variable.targetLength);   \
-    PARSE_SUB_ARGS("strategy=",        "strat=", variable.strategy);       \
+    PARSE_SUB_ARGS("windowLog=",       "wlog=",  variable.vals[wlog_ind]); \
+    PARSE_SUB_ARGS("chainLog=" ,       "clog=",  variable.vals[clog_ind]); \
+    PARSE_SUB_ARGS("hashLog=",         "hlog=",  variable.vals[hlog_ind]); \
+    PARSE_SUB_ARGS("searchLog=" ,      "slog=",  variable.vals[slog_ind]); \
+    PARSE_SUB_ARGS("searchLength=",    "slen=",  variable.vals[slen_ind]); \
+    PARSE_SUB_ARGS("targetLength=" ,   "tlen=",  variable.vals[tlen_ind]); \
+    PARSE_SUB_ARGS("strategy=",        "strat=", variable.vals[strt_ind]); \
+    PARSE_SUB_ARGS("forceAttachDict=", "fad="  , variable.vals[fadt_ind]); \
 }
 
 int main(int argc, const char** argv)
@@ -2426,12 +2475,11 @@ int main(int argc, const char** argv)
     const char* input_filename = NULL;
     const char* dictFileName = NULL;
     U32 main_pause = 0;
-    int cLevel = 0;
+    int cLevelOpt = 0, cLevelRun = 0;
     int seperateFiles = 0;
-
     constraint_t target = { 0, 0, (U32)-1 }; 
 
-    ZSTD_compressionParameters paramTarget = emptyParams();
+    paramValues_t paramTarget = emptyParams();
     g_params = emptyParams();
 
     assert(argc>=1);   /* for exename */
@@ -2441,9 +2489,7 @@ int main(int argc, const char** argv)
 
     for(i=1; i
Date: Tue, 14 Aug 2018 14:44:47 -0700
Subject: [PATCH 163/372] Fix scan-build warnings in bench.c

---
 programs/bench.c | 18 +++++++++++-------
 1 file changed, 11 insertions(+), 7 deletions(-)

diff --git a/programs/bench.c b/programs/bench.c
index 7662678ab..79ef42caa 100644
--- a/programs/bench.c
+++ b/programs/bench.c
@@ -597,15 +597,16 @@ static BMK_return_t BMK_benchMemAdvancedNoAlloc(
                         results.error = compressionResults.error;
                         return results;
                     }
+
                     if(compressionResults.result.nanoSecPerRun == 0) {
                         results.result.cSpeed = 0;
                     } else {
                         results.result.cSpeed = srcSize * TIMELOOP_NANOSEC / compressionResults.result.nanoSecPerRun;
                     }
+
                     results.result.cSize = compressionResults.result.sumOfReturn;
                     {   
                         int const ratioAccuracy = (ratio < 10.) ? 3 : 2;
-                        results.result.cSpeed = (srcSize * TIMELOOP_NANOSEC / compressionResults.result.nanoSecPerRun);
                         cSize = compressionResults.result.sumOfReturn;
                         results.result.cSize = cSize;
                         ratio = (double)srcSize / results.result.cSize;
@@ -626,6 +627,7 @@ static BMK_return_t BMK_benchMemAdvancedNoAlloc(
                         results.error = decompressionResults.error;
                         return results;
                     }
+
                     if(decompressionResults.result.nanoSecPerRun == 0) {
                         results.result.dSpeed = 0;
                     } else {
@@ -634,7 +636,6 @@ static BMK_return_t BMK_benchMemAdvancedNoAlloc(
 
                     {   
                         int const ratioAccuracy = (ratio < 10.) ? 3 : 2;
-                        results.result.dSpeed = (srcSize * TIMELOOP_NANOSEC/ decompressionResults.result.nanoSecPerRun);
                         markNb = (markNb+1) % NB_MARKS;
                         DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.*f),%6.*f MB/s ,%6.1f MB/s \r",
                                 marks[markNb], displayName, (U32)srcSize, (U32)results.result.cSize,
@@ -737,14 +738,16 @@ BMK_return_t BMK_benchMemAdvanced(const void* srcBuffer, size_t srcSize,
     void* const internalDstBuffer = dstBuffer ? NULL : malloc(maxCompressedSize);
     void* const compressedBuffer = dstBuffer ? dstBuffer : internalDstBuffer;
 
-    void* resultBuffer = malloc(srcSize);
-
     BMK_return_t results = { { 0, 0, 0, 0 }, 0 };
+
+    int parametersConflict = !dstBuffer ^ !dstCapacity;
+
+    void* resultBuffer = srcSize ? malloc(srcSize) : NULL;
+
     int allocationincomplete = !srcPtrs || !srcSizes || !cPtrs || 
         !cSizes || !cCapacities || !resPtrs || !resSizes || 
         !timeStateCompress || !timeStateDecompress || !compressedBuffer || !resultBuffer;
 
-    int parametersConflict = !dstBuffer ^ !dstCapacity;
 
 
     if (!allocationincomplete && !parametersConflict) {
@@ -809,7 +812,7 @@ static size_t BMK_findMaxMem(U64 requiredMem)
     do {
         testmem = (BYTE*)malloc((size_t)requiredMem);
         requiredMem -= step;
-    } while (!testmem);
+    } while (!testmem && requiredMem > 0);
 
     free(testmem);
     return (size_t)(requiredMem);
@@ -937,7 +940,8 @@ BMK_return_t BMK_benchFilesAdvanced(const char* const * const fileNamesTable, un
     if ((U64)benchedSize > totalSizeToLoad) benchedSize = (size_t)totalSizeToLoad;
     if (benchedSize < totalSizeToLoad)
         DISPLAY("Not enough memory; testing %u MB only...\n", (U32)(benchedSize >> 20));
-    srcBuffer = malloc(benchedSize);
+
+    srcBuffer = benchedSize ? malloc(benchedSize) : NULL;
     if (!srcBuffer) {
         free(dictBuffer);
         free(fileSizes);

From 96725989ef376b4fec697958837216f66d962406 Mon Sep 17 00:00:00 2001
From: George Lu 
Date: Thu, 9 Aug 2018 16:36:34 -0700
Subject: [PATCH 164/372] Temp fix perf regression

---
 tests/paramgrill.c | 22 +++++++++++++++++-----
 1 file changed, 17 insertions(+), 5 deletions(-)

diff --git a/tests/paramgrill.c b/tests/paramgrill.c
index aab46ede5..4324678ad 100644
--- a/tests/paramgrill.c
+++ b/tests/paramgrill.c
@@ -185,9 +185,9 @@ typedef struct {
     U32 vals[NUM_PARAMS];
 } paramValues_t;
 
-//TODO: unset -> 0?
 static ZSTD_compressionParameters pvalsToCParams(paramValues_t p) {
     ZSTD_compressionParameters c;
+    memset(&c, 0, sizeof(ZSTD_compressionParameters));
     c.windowLog = p.vals[wlog_ind];
     c.chainLog = p.vals[clog_ind];
     c.hashLog = p.vals[hlog_ind];
@@ -383,11 +383,9 @@ static int paramValid(paramValues_t paramTarget) {
     for(i = 0; i < NUM_PARAMS; i++) {
         CLAMPCHECK(paramTarget.vals[i], mintable[i], maxtable[i]);
     }
-    //TODO: Strategy could be valid at 0 before, is that right?
     return 1;
 }
 
-//TODO: doesn't affect strategy?
 static paramValues_t cParamUnsetMin(paramValues_t paramTarget) {
     varInds_t i;
     for(i = 0; i < NUM_PARAMS; i++) {
@@ -1399,7 +1397,7 @@ static int variableParams(const paramValues_t paramConstraints, varInds_t* res,
 /* slight change from old paramVariation, targetLength can only take on powers of 2 now (999 ~= 1024?) */
 /* take max/min bounds into account as well? */
 static void paramVaryOnce(const varInds_t paramIndex, const int amt, paramValues_t* ptr) {
-    ptr->vals[paramIndex] = rangeMap(paramIndex, invRangeMap(paramIndex, ptr->vals[paramIndex]) + amt); //TODO: bounds check. 
+    ptr->vals[paramIndex] = rangeMap(paramIndex, invRangeMap(paramIndex, ptr->vals[paramIndex]) + amt);
 }
 
 /* varies ptr by nbChanges respecting varyParams*/
@@ -1447,10 +1445,23 @@ static unsigned memoTableInd(const paramValues_t* ptr, const varInds_t* varyPara
 static void memoTableIndInv(paramValues_t* ptr, const varInds_t* varyParams, const int varyLen, size_t ind) {
     int i;
     for(i = varyLen - 1; i >= 0; i--) {
+        /* This is cleaner/easier to generalize but slower
         varInds_t v = varyParams[i];
         if(v == strt_ind) continue;
         ptr->vals[v] = rangeMap(v, ind % rangetable[v]);
-        ind /= rangetable[v];
+        ind /= rangetable[v]; */
+
+        switch(varyParams[i]) {
+            case wlog_ind: ptr->vals[wlog_ind] = ind % rangetable[wlog_ind] + mintable[wlog_ind];       ind /= rangetable[wlog_ind]; break;
+            case clog_ind: ptr->vals[clog_ind] = ind % rangetable[clog_ind] + mintable[clog_ind];       ind /= rangetable[clog_ind]; break;
+            case hlog_ind: ptr->vals[hlog_ind] = ind % rangetable[hlog_ind] + mintable[hlog_ind];       ind /= rangetable[hlog_ind]; break;
+            case slog_ind: ptr->vals[slog_ind] = ind % rangetable[slog_ind] + mintable[slog_ind];       ind /= rangetable[slog_ind]; break;
+            case slen_ind: ptr->vals[slen_ind] = ind % rangetable[slen_ind] + mintable[slen_ind];       ind /= rangetable[slen_ind]; break;
+            case tlen_ind: ptr->vals[tlen_ind] = tlen_table[(ind % rangetable[tlen_ind])];              ind /= rangetable[tlen_ind]; break;
+            case fadt_ind: ptr->vals[fadt_ind] = ind % rangetable[fadt_ind] - 1;                        ind /= rangetable[fadt_ind]; break;
+            case strt_ind:
+            case NUM_PARAMS: break;
+        }
     }
 }
 
@@ -1471,6 +1482,7 @@ static void initMemoTable(U8* memoTable, paramValues_t paramConstraints, const c
     memset(memoTable, 0, arrayLen);
     paramConstraints = cParamUnsetMin(paramConstraints);
 
+
     for(i = 0; i < arrayLen; i++) {
         memoTableIndInv(¶mConstraints, varyParams, varyLen, i);
         if(ZSTD_estimateCStreamSize_usingCParams(pvalsToCParams(paramConstraints)) > (size_t)target.cMem) {

From 8c918edd3a87a006200f704b9119434c5f3250f3 Mon Sep 17 00:00:00 2001
From: George Lu 
Date: Fri, 10 Aug 2018 16:14:12 -0700
Subject: [PATCH 165/372] MAke it easier to add params

Make memoTable size limited
---
 tests/paramgrill.c | 499 ++++++++++++++++++++++++---------------------
 1 file changed, 272 insertions(+), 227 deletions(-)

diff --git a/tests/paramgrill.c b/tests/paramgrill.c
index 4324678ad..c2979cb83 100644
--- a/tests/paramgrill.c
+++ b/tests/paramgrill.c
@@ -86,7 +86,15 @@ static const int g_maxNbVariations = 64;
 #define CHECKTIME(r) { if(BMK_timeSpan(g_time) > g_timeLimit_s) { DEBUGOUTPUT("Time Limit Reached\n"); return r; } }
 #define CHECKTIMEGT(ret, val, _gototag) {if(BMK_timeSpan(g_time) > g_timeLimit_s) { DEBUGOUTPUT("Time Limit Reached\n"); ret = val; goto _gototag; } }
 
-#define PARAM_UNSET ((U32)-2) /* can't be -1 b/c fadt */
+#define PARAM_UNSET ((U32)-2) /* can't be -1 b/c fadt uses -1 */
+
+static const char* g_stratName[ZSTD_btultra+1] = {
+                "(none)       ", "ZSTD_fast    ", "ZSTD_dfast   ",
+                "ZSTD_greedy  ", "ZSTD_lazy    ", "ZSTD_lazy2   ",
+                "ZSTD_btlazy2 ", "ZSTD_btopt   ", "ZSTD_btultra "};
+
+
+static const U32 tlen_table[TLEN_RANGE] = { 0, 1, 2, 4, 6, 8, 12, 16, 24, 32, 48, 64, 96, 128, 256, 512, 999 };
 
 /*-************************************
 *  Setup for Adding new params
@@ -105,6 +113,14 @@ typedef enum {
     NUM_PARAMS = 8
 } varInds_t;
 
+typedef struct {
+    U32 vals[NUM_PARAMS];
+} paramValues_t;
+
+/* list of parameters */
+static const varInds_t paramTable[NUM_PARAMS] = 
+        { wlog_ind, clog_ind, hlog_ind, slog_ind, slen_ind, tlen_ind, strt_ind, fadt_ind };
+
 /* maximum value of parameters */
 static const U32 mintable[NUM_PARAMS] = 
         { ZSTD_WINDOWLOG_MIN, ZSTD_CHAINLOG_MIN, ZSTD_HASHLOG_MIN, ZSTD_SEARCHLOG_MIN, ZSTD_SEARCHLENGTH_MIN, ZSTD_TARGETLENGTH_MIN, ZSTD_fast, FADT_MIN };
@@ -117,15 +133,19 @@ static const U32 maxtable[NUM_PARAMS] =
 static const U32 rangetable[NUM_PARAMS] = 
         { WLOG_RANGE, CLOG_RANGE, HLOG_RANGE, SLOG_RANGE, SLEN_RANGE, TLEN_RANGE, STRT_RANGE, FADT_RANGE };
 
+/* ZSTD_cctxSetParameter() index to set */
 static const ZSTD_cParameter cctxSetParamTable[NUM_PARAMS] = 
         { ZSTD_p_windowLog, ZSTD_p_chainLog, ZSTD_p_hashLog, ZSTD_p_searchLog, ZSTD_p_minMatch, ZSTD_p_targetLength, ZSTD_p_compressionStrategy, ZSTD_p_forceAttachDict };
 
+/* names of parameters */
 static const char* g_paramNames[NUM_PARAMS] = 
-        { "windowLog", "chainLog", "hashLog","searchLog", "searchLength", "targetLength", "strategy", "forceAttachDict"};
+        { "windowLog", "chainLog", "hashLog","searchLog", "searchLength", "targetLength", "strategy", "forceAttachDict" };
 
+/* shortened names of parameters */
+static const char* g_shortParamNames[NUM_PARAMS] = 
+        { "wlog", "clog", "hlog","slog", "slen", "tlen", "strt", "fadt" };
 
-static const U32 tlen_table[TLEN_RANGE] = { 0, 1, 2, 4, 6, 8, 12, 16, 24, 32, 48, 64, 96, 128, 256, 512, 999 };
-/* maps value from 0 to rangetable[param] - 1 to valid paramvalue */
+/* maps value from { 0 to rangetable[param] - 1 } to valid paramvalues */
 static U32 rangeMap(varInds_t param, U32 ind) {
     ind = MIN(ind, rangetable[param] - 1);
     switch(param) {
@@ -141,6 +161,7 @@ static U32 rangeMap(varInds_t param, U32 ind) {
         case strt_ind:
             return mintable[param] + ind;
         case NUM_PARAMS:
+            DISPLAY("Error, not a valid param\n ");
             return (U32)-1;
     }
     return 0; /* should never happen, stop compiler warnings */
@@ -176,49 +197,26 @@ static U32 invRangeMap(varInds_t param, U32 value) {
         case strt_ind:
             return value - mintable[param];
         case NUM_PARAMS:
+            DISPLAY("Error, not a valid param\n ");
             return (U32)-1;
     }
     return 0; /* should never happen, stop compiler warnings */
 }
 
-typedef struct {
-    U32 vals[NUM_PARAMS];
-} paramValues_t;
-
-static ZSTD_compressionParameters pvalsToCParams(paramValues_t p) {
-    ZSTD_compressionParameters c;
-    memset(&c, 0, sizeof(ZSTD_compressionParameters));
-    c.windowLog = p.vals[wlog_ind];
-    c.chainLog = p.vals[clog_ind];
-    c.hashLog = p.vals[hlog_ind];
-    c.searchLog = p.vals[slog_ind];
-    c.searchLength = p.vals[slen_ind];
-    c.targetLength = p.vals[tlen_ind];
-    c.strategy = p.vals[strt_ind];
-    /* no forceAttachDict */
-    return c;
-}
-
-/* 0 = auto for fadt */
-static paramValues_t cParamsToPVals(ZSTD_compressionParameters c) {
-    paramValues_t p;
-    p.vals[wlog_ind] = c.windowLog;
-    p.vals[clog_ind] = c.chainLog;
-    p.vals[hlog_ind] = c.hashLog;
-    p.vals[slog_ind] = c.searchLog;
-    p.vals[slen_ind] = c.searchLength;
-    p.vals[tlen_ind] = c.targetLength;
-    p.vals[strt_ind] = c.strategy;
-    p.vals[fadt_ind] = 0;
-    return p;
-}
-
-/* equivalent of ZSTD_adjustCParams for paramValues_t */
-static paramValues_t adjustParams(paramValues_t p, size_t maxBlockSize, size_t dictSize) {
-    U32 fval = p.vals[fadt_ind];
-    p = cParamsToPVals(ZSTD_adjustCParams(pvalsToCParams(p), maxBlockSize, dictSize));
-    p.vals[fadt_ind] = fval;
-    return p;
+/* display of params */
+static void displayParamVal(FILE* f, varInds_t param, U32 value, int width) {
+    switch(param) {
+        case fadt_ind: if(width) { fprintf(f, "%*d", width, (int)value); } else { fprintf(f, "%d", (int)value); } break;
+        case strt_ind: if(width) { fprintf(f, "%*s", width, g_stratName[value]); } else { fprintf(f, "%s", g_stratName[value]); } break;
+        case wlog_ind: 
+        case clog_ind: 
+        case hlog_ind: 
+        case slog_ind: 
+        case slen_ind: 
+        case tlen_ind: if(width) { fprintf(f, "%*u", width, value); } else { fprintf(f, "%u", value); } break;
+        case NUM_PARAMS:
+            DISPLAY("Error, not a valid param\n "); break;
+    }
 }
 
 /*-************************************
@@ -238,7 +236,7 @@ static U32 g_target = 0;
 static U32 g_noSeed = 0;
 static paramValues_t g_params; /* Initialized at the beginning of main w/ emptyParams() function */
 static UTIL_time_t g_time; /* to be used to compare solution finding speeds to compare to original */
-
+static U32 g_memoLimit = (U32)-1; //32 MB
 
 typedef struct {
     BMK_result_t result;
@@ -293,6 +291,51 @@ void BMK_SetNbIterations(int nbLoops)
 *  Private functions
 *********************************************************/
 
+static ZSTD_compressionParameters pvalsToCParams(paramValues_t p) {
+    ZSTD_compressionParameters c;
+    memset(&c, 0, sizeof(ZSTD_compressionParameters));
+    c.windowLog = p.vals[wlog_ind];
+    c.chainLog = p.vals[clog_ind];
+    c.hashLog = p.vals[hlog_ind];
+    c.searchLog = p.vals[slog_ind];
+    c.searchLength = p.vals[slen_ind];
+    c.targetLength = p.vals[tlen_ind];
+    c.strategy = p.vals[strt_ind];
+    /* no forceAttachDict */
+    return c;
+}
+
+static paramValues_t cParamsToPVals(ZSTD_compressionParameters c) {
+    paramValues_t p;
+    varInds_t i;
+    p.vals[wlog_ind] = c.windowLog;
+    p.vals[clog_ind] = c.chainLog;
+    p.vals[hlog_ind] = c.hashLog;
+    p.vals[slog_ind] = c.searchLog;
+    p.vals[slen_ind] = c.searchLength;
+    p.vals[tlen_ind] = c.targetLength;
+    p.vals[strt_ind] = c.strategy;
+
+    /* set all other params to their minimum value */
+    for(i = strt_ind + 1; i < NUM_PARAMS; i++) {
+        p.vals[i] = mintable[i];
+    }
+    return p;
+}
+
+/* equivalent of ZSTD_adjustCParams for paramValues_t */
+static paramValues_t adjustParams(paramValues_t p, size_t maxBlockSize, size_t dictSize) {
+    paramValues_t ot = p;
+    varInds_t i;
+    p = cParamsToPVals(ZSTD_adjustCParams(pvalsToCParams(p), maxBlockSize, dictSize));
+
+    /* retain value of all other parameters */
+    for(i = strt_ind + 1; i < NUM_PARAMS; i++) {
+        p.vals[i] = ot.vals[i];
+    }
+    return p;
+}
+
 /* accuracy in seconds only, span can be multiple years */
 static U32 BMK_timeSpan(UTIL_time_t tStart) { return (U32)(UTIL_clockSpanMicro(tStart) / 1000000ULL); }
 
@@ -375,9 +418,6 @@ static void findClockGranularity(void) {
         return 0;                                     \
 }   }
 
-
-/* Like ZSTD_checkCParams() but allows 0's */
-/* no check on targetLen? */
 static int paramValid(paramValues_t paramTarget) {
     U32 i;
     for(i = 0; i < NUM_PARAMS; i++) {
@@ -400,8 +440,11 @@ static void BMK_translateAdvancedParams(FILE* f, const paramValues_t params) {
     U32 i;
     fprintf(f,"--zstd=");
     for(i = 0; i < NUM_PARAMS; i++) {
-        fprintf(f,"%s", g_paramNames[i]);
-        fprintf(f,"=%u", params.vals[i]);
+        fprintf(f,"%s=", g_paramNames[i]);
+        
+        if(i == strt_ind) { fprintf(f,"%u", params.vals[i]); }
+        else { displayParamVal(f, i, params.vals[i], 0); }
+
         if(i != NUM_PARAMS - 1) {
             fprintf(f, ",");
         }
@@ -409,19 +452,16 @@ static void BMK_translateAdvancedParams(FILE* f, const paramValues_t params) {
     fprintf(f, "\n");
 }
 
-
-static const char* g_stratName[ZSTD_btultra+1] = {
-                "(none)       ", "ZSTD_fast    ", "ZSTD_dfast   ",
-                "ZSTD_greedy  ", "ZSTD_lazy    ", "ZSTD_lazy2   ",
-                "ZSTD_btlazy2 ", "ZSTD_btopt   ", "ZSTD_btultra "};
-
 static void BMK_displayOneResult(FILE* f, winnerInfo_t res, size_t srcSize) {
+            varInds_t v;
             res.params = cParamUnsetMin(res.params);
-            fprintf(f,"    {%3u,%3u,%3u,%3u,%3u,%3u,%3d, %s},  ",
-                res.params.vals[wlog_ind], res.params.vals[clog_ind], res.params.vals[hlog_ind], res.params.vals[slog_ind], res.params.vals[slen_ind],
-                res.params.vals[tlen_ind], (int)res.params.vals[fadt_ind], g_stratName[res.params.vals[strt_ind]]);
-            fprintf(f,
-            "   /* R:%5.3f at %5.1f MB/s - %5.1f MB/s */\n",
+            fprintf(f,"    {");
+            for(v = 0; v < NUM_PARAMS; v++) {
+                if(v != 0) { fprintf(f, ","); }
+                displayParamVal(f, v, res.params.vals[v], 3);
+            }
+
+            fprintf(f, "},     /* R:%5.3f at %5.1f MB/s - %5.1f MB/s */\n",
             (double)srcSize / res.result.cSize, (double)res.result.cSpeed / (1 MB), (double)res.result.dSpeed / (1 MB));
 }
 
@@ -494,7 +534,7 @@ static paramValues_t emptyParams(void) {
     return p;
 }
 
-static winnerInfo_t initWinnerInfo(paramValues_t p) {
+static winnerInfo_t initWinnerInfo(const paramValues_t p) {
     winnerInfo_t w1;
     w1.result.cSpeed = 0.;
     w1.result.dSpeed = 0.;
@@ -515,6 +555,7 @@ typedef struct {
     void** resPtrs;
     size_t* resSizes;
     size_t nbBlocks;
+    size_t maxBlockSize;
 } buffers_t;
 
 typedef struct {
@@ -528,27 +569,6 @@ typedef struct {
 *  From bench.c
 *********************************************************/
 
-static void BMK_initCCtx(ZSTD_CCtx* ctx, 
-    const void* dictBuffer, const size_t dictBufferSize, const int cLevel, 
-    const paramValues_t* comprParams) {
-    varInds_t i;
-    ZSTD_CCtx_reset(ctx);
-    ZSTD_CCtx_resetParameters(ctx);
-    ZSTD_CCtx_setParameter(ctx, ZSTD_p_compressionLevel, cLevel);
-
-    for(i = 0; i < NUM_PARAMS; i++) {
-        if(comprParams->vals[i] != PARAM_UNSET)
-        ZSTD_CCtx_setParameter(ctx, cctxSetParamTable[i], comprParams->vals[i]);
-    }
-    ZSTD_CCtx_loadDictionary(ctx, dictBuffer, dictBufferSize);
-}
-
-static void BMK_initDCtx(ZSTD_DCtx* dctx, 
-    const void* dictBuffer, const size_t dictBufferSize) {
-    ZSTD_DCtx_reset(dctx);
-    ZSTD_DCtx_loadDictionary(dctx, dictBuffer, dictBufferSize);
-}
-
 typedef struct {
     ZSTD_CCtx* ctx;
     const void* dictBuffer;
@@ -559,7 +579,17 @@ typedef struct {
 
 static size_t local_initCCtx(void* payload) {
     const BMK_initCCtxArgs* ag = (const BMK_initCCtxArgs*)payload;
-    BMK_initCCtx(ag->ctx, ag->dictBuffer, ag->dictBufferSize, ag->cLevel, ag->comprParams);
+    varInds_t i;
+    ZSTD_CCtx_reset(ag->ctx);
+    ZSTD_CCtx_resetParameters(ag->ctx);
+    ZSTD_CCtx_setParameter(ag->ctx, ZSTD_p_compressionLevel, ag->cLevel);
+
+    for(i = 0; i < NUM_PARAMS; i++) {
+        if(ag->comprParams->vals[i] != PARAM_UNSET)
+        ZSTD_CCtx_setParameter(ag->ctx, cctxSetParamTable[i], ag->comprParams->vals[i]);
+    }
+    ZSTD_CCtx_loadDictionary(ag->ctx, ag->dictBuffer, ag->dictBufferSize);
+
     return 0;
 }
 
@@ -571,7 +601,8 @@ typedef struct {
 
 static size_t local_initDCtx(void* payload) {
     const BMK_initDCtxArgs* ag = (const BMK_initDCtxArgs*)payload;
-    BMK_initDCtx(ag->dctx, ag->dictBuffer, ag->dictBufferSize);
+    ZSTD_DCtx_reset(ag->dctx);
+    ZSTD_DCtx_loadDictionary(ag->dctx, ag->dictBuffer, ag->dictBufferSize);
     return 0;
 }
 
@@ -636,6 +667,72 @@ static size_t local_defaultDecompress(
 *  From bench.c End
 *********************************************************/
 
+static void optimizerAdjustInput(paramValues_t* pc, const size_t maxBlockSize) {
+    varInds_t v;
+    for(v = 0; v < NUM_PARAMS; v++) {
+        if(pc->vals[v] != PARAM_UNSET) {
+            U32 newval = MIN(MAX(pc->vals[v], mintable[v]), maxtable[v]);
+            if(newval != pc->vals[v]) {
+                pc->vals[v] = newval;
+                DISPLAY("Warning: parameter %s not in valid range, adjusting to ", g_paramNames[v]); displayParamVal(stderr, v, newval, 0); DISPLAY("\n");
+            }
+        }
+    }
+
+    if(pc->vals[wlog_ind] != PARAM_UNSET) {
+
+        U32 sshb = maxBlockSize > 1 ? ZSTD_highbit32((U32)(maxBlockSize-1)) + 1 : 1;
+        /* edge case of highBit not working for 0 */
+
+        if(maxBlockSize < (1ULL << 31) && sshb + 1 < pc->vals[wlog_ind]) {
+            U32 adjust = MAX(mintable[wlog_ind], sshb);
+            if(adjust != pc->vals[wlog_ind]) {
+                pc->vals[wlog_ind] = adjust;
+                DISPLAY("Warning: windowLog larger than src/block size, adjusted to %u\n", pc->vals[wlog_ind]);
+            }
+        }
+    }
+
+    if(pc->vals[wlog_ind] != PARAM_UNSET && pc->vals[clog_ind] != PARAM_UNSET) {
+        U32 maxclog;
+        if(pc->vals[strt_ind] == PARAM_UNSET || pc->vals[strt_ind] >= (U32)ZSTD_btlazy2) {
+            maxclog = pc->vals[wlog_ind] + 1;
+        } else {
+            maxclog = pc->vals[wlog_ind];
+        }
+
+        if(pc->vals[clog_ind] > maxclog) {
+            pc->vals[clog_ind] = maxclog;
+            DISPLAY("Warning: chainlog too much larger than windowLog size, adjusted to %u\n", pc->vals[clog_ind]);
+        }
+    }
+
+    if(pc->vals[wlog_ind] != PARAM_UNSET && pc->vals[hlog_ind] != PARAM_UNSET) {
+        if(pc->vals[wlog_ind] + 1 < pc->vals[hlog_ind]) {
+            pc->vals[hlog_ind] = pc->vals[wlog_ind] + 1;
+            DISPLAY("Warning: hashlog too much larger than windowLog size, adjusted to %u\n", pc->vals[hlog_ind]);
+        }
+    }
+    
+    if(pc->vals[slog_ind] != PARAM_UNSET && pc->vals[clog_ind] != PARAM_UNSET) {
+        if(pc->vals[slog_ind] > pc->vals[clog_ind]) {
+            pc->vals[clog_ind] = pc->vals[slog_ind];
+            DISPLAY("Warning: searchLog larger than chainLog, adjusted to %u\n", pc->vals[slog_ind]);
+        }
+    }
+}
+
+/* what about low something like clog vs hlog in lvl 1?  */
+static int redundantParams(const paramValues_t paramValues, const constraint_t target, const size_t srcSize) {
+    return 
+       (ZSTD_estimateCStreamSize_usingCParams(pvalsToCParams(paramValues)) > (size_t)target.cMem) /* Uses too much memory */
+    || ((1ULL << (paramValues.vals[wlog_ind] - 1)) >= srcSize && paramValues.vals[wlog_ind] != mintable[wlog_ind]) /* wlog too much bigger than src size */
+    || (paramValues.vals[clog_ind] > (paramValues.vals[wlog_ind] + (paramValues.vals[strt_ind] > ZSTD_btlazy2))) /* chainLog larger than windowLog*/
+    || (paramValues.vals[slog_ind] > paramValues.vals[clog_ind]) /* searchLog larger than chainLog */
+    || (paramValues.vals[hlog_ind] > paramValues.vals[wlog_ind] + 1); /* hashLog larger than windowLog + 1 */
+    
+}
+
 static void freeNonSrcBuffers(const buffers_t b) {
     free(b.srcPtrs);
     free(b.srcSizes);
@@ -722,6 +819,7 @@ static int createBuffersFromMemory(buffers_t* buff, void * srcBuffer, size_t nbF
     buff->dstCapacities[0] = ZSTD_compressBound(buff->srcSizes[0]);
     buff->dstSizes[0] = buff->dstCapacities[0];
     buff->resSizes[0] = buff->srcSizes[0];
+    buff->maxBlockSize = buff->srcSizes[0];
 
     for(n = 1; n < blockNb; n++) {
         buff->dstPtrs[n] = ((char*)buff->dstPtrs[n-1]) + buff->dstCapacities[n-1];
@@ -729,6 +827,8 @@ static int createBuffersFromMemory(buffers_t* buff, void * srcBuffer, size_t nbF
         buff->dstCapacities[n] = ZSTD_compressBound(buff->srcSizes[n]);
         buff->dstSizes[n] = buff->dstCapacities[n];
         buff->resSizes[n] = buff->srcSizes[n];
+
+        buff->maxBlockSize = MAX(buff->maxBlockSize, buff->srcSizes[n]);
     }
 
     buff->nbBlocks = blockNb;
@@ -786,7 +886,7 @@ static int createBuffers(buffers_t* buff, const char* const * const fileNamesTab
             char* buffer = (char*)(srcBuffer); 
             size_t const readSize = fread((buffer)+pos, 1, (size_t)fileSize, f);
             fclose(f);
-            if (readSize != (size_t)fileSize) { /* should we accept partial read? */
+            if (readSize != (size_t)fileSize) {
                 DISPLAY("could not read %s", fileNamesTable[n]);
                 ret = 1;
                 goto _cleanUp;
@@ -992,7 +1092,7 @@ static BMK_return_t BMK_benchMemInvertible(const buffers_t buf, const contexts_t
 }
 
 static int BMK_benchParam(BMK_result_t* resultPtr,
-                buffers_t buf, contexts_t ctx,
+                const buffers_t buf, const contexts_t ctx,
                 const paramValues_t cParams) {
     BMK_return_t res = BMK_benchMemInvertible(buf, ctx, BASE_CLEVEL, &cParams, BMK_both, BMK_timeMode, 3);
     *resultPtr = res.result;
@@ -1009,7 +1109,7 @@ static int BMK_benchParam(BMK_result_t* resultPtr,
 #define SPEED_RESULT 4
 #define SIZE_RESULT 5
 /* maybe have epsilon-eq to limit table size? */
-static int speedSizeCompare(BMK_result_t r1, BMK_result_t r2) {
+static int speedSizeCompare(const BMK_result_t r1, const BMK_result_t r2) {
     if(r1.cSpeed < r2.cSpeed) {
         if(r1.cSize >= r2.cSize) {
             return BETTER_RESULT;
@@ -1025,7 +1125,7 @@ static int speedSizeCompare(BMK_result_t r1, BMK_result_t r2) {
 
 /* 0 for insertion, 1 for no insert */
 /* maintain invariant speedSizeCompare(n, n->next) = SPEED_RESULT */
-static int insertWinner(winnerInfo_t w, constraint_t targetConstraints) {
+static int insertWinner(const winnerInfo_t w, const constraint_t targetConstraints) {
     BMK_result_t r = w.result;
     winner_ll_node* cur_node = g_winners;
     /* first node to insert */
@@ -1194,7 +1294,7 @@ static void BMK_printWinnerOpt(FILE* f, const U32 cLevel, const BMK_result_t res
     }
 }
 
-static void BMK_printWinners2(FILE* f, const oldWinnerInfo_t* winners, size_t srcSize)
+static void BMK_printWinners2(FILE* f, const oldWinnerInfo_t* winners, const size_t srcSize)
 {
     int cLevel;
 
@@ -1206,7 +1306,7 @@ static void BMK_printWinners2(FILE* f, const oldWinnerInfo_t* winners, size_t sr
 }
 
 
-static void BMK_printWinners(FILE* f, const oldWinnerInfo_t* winners, size_t srcSize)
+static void BMK_printWinners(FILE* f, const oldWinnerInfo_t* winners, const size_t srcSize)
 {
     fseek(f, 0, SEEK_SET);
     BMK_printWinners2(f, winners, srcSize);
@@ -1244,7 +1344,7 @@ static void BMK_init_level_constraints(int bytePerSec_level1)
 }
 
 static int BMK_seed(oldWinnerInfo_t* winners, const ZSTD_compressionParameters params, 
-                    buffers_t buf, contexts_t ctx)
+                    const buffers_t buf, const contexts_t ctx)
 {
     BMK_result_t testResult;
     int better = 0;
@@ -1422,9 +1522,8 @@ static size_t memoTableLen(const varInds_t* varyParams, const int varyLen) {
     size_t arrayLen = 1;
     int i;
     for(i = 0; i < varyLen; i++) {
-        if(varyParams[i] != strt_ind) {
-            arrayLen *= rangetable[varyParams[i]];
-        }
+        if(varyParams[i] == strt_ind) continue; /* strategy separated by table */
+        arrayLen *= rangetable[varyParams[i]];
     }
     return arrayLen;
 }
@@ -1452,13 +1551,13 @@ static void memoTableIndInv(paramValues_t* ptr, const varInds_t* varyParams, con
         ind /= rangetable[v]; */
 
         switch(varyParams[i]) {
-            case wlog_ind: ptr->vals[wlog_ind] = ind % rangetable[wlog_ind] + mintable[wlog_ind];       ind /= rangetable[wlog_ind]; break;
-            case clog_ind: ptr->vals[clog_ind] = ind % rangetable[clog_ind] + mintable[clog_ind];       ind /= rangetable[clog_ind]; break;
-            case hlog_ind: ptr->vals[hlog_ind] = ind % rangetable[hlog_ind] + mintable[hlog_ind];       ind /= rangetable[hlog_ind]; break;
-            case slog_ind: ptr->vals[slog_ind] = ind % rangetable[slog_ind] + mintable[slog_ind];       ind /= rangetable[slog_ind]; break;
-            case slen_ind: ptr->vals[slen_ind] = ind % rangetable[slen_ind] + mintable[slen_ind];       ind /= rangetable[slen_ind]; break;
-            case tlen_ind: ptr->vals[tlen_ind] = tlen_table[(ind % rangetable[tlen_ind])];              ind /= rangetable[tlen_ind]; break;
-            case fadt_ind: ptr->vals[fadt_ind] = ind % rangetable[fadt_ind] - 1;                        ind /= rangetable[fadt_ind]; break;
+            case wlog_ind: ptr->vals[wlog_ind] = ind % rangetable[wlog_ind] + mintable[wlog_ind]; ind /= rangetable[wlog_ind]; break;
+            case clog_ind: ptr->vals[clog_ind] = ind % rangetable[clog_ind] + mintable[clog_ind]; ind /= rangetable[clog_ind]; break;
+            case hlog_ind: ptr->vals[hlog_ind] = ind % rangetable[hlog_ind] + mintable[hlog_ind]; ind /= rangetable[hlog_ind]; break;
+            case slog_ind: ptr->vals[slog_ind] = ind % rangetable[slog_ind] + mintable[slog_ind]; ind /= rangetable[slog_ind]; break;
+            case slen_ind: ptr->vals[slen_ind] = ind % rangetable[slen_ind] + mintable[slen_ind]; ind /= rangetable[slen_ind]; break;
+            case tlen_ind: ptr->vals[tlen_ind] = tlen_table[(ind % rangetable[tlen_ind])];        ind /= rangetable[tlen_ind]; break;
+            case fadt_ind: ptr->vals[fadt_ind] = ind % rangetable[fadt_ind] - 1;                  ind /= rangetable[fadt_ind]; break;
             case strt_ind:
             case NUM_PARAMS: break;
         }
@@ -1473,58 +1572,19 @@ static void memoTableIndInv(paramValues_t* ptr, const varInds_t* varyParams, con
 static void initMemoTable(U8* memoTable, paramValues_t paramConstraints, const constraint_t target, const varInds_t* varyParams, const int varyLen, const size_t srcSize) {
     size_t i;
     size_t arrayLen = memoTableLen(varyParams, varyLen);
-    int cwFixed = paramConstraints.vals[clog_ind] == PARAM_UNSET || paramConstraints.vals[wlog_ind] == PARAM_UNSET;
-    int scFixed = paramConstraints.vals[slog_ind] == PARAM_UNSET || paramConstraints.vals[clog_ind] == PARAM_UNSET;
-    int whFixed = paramConstraints.vals[wlog_ind] == PARAM_UNSET || paramConstraints.vals[hlog_ind] == PARAM_UNSET;
-    int wFixed = paramConstraints.vals[wlog_ind] == PARAM_UNSET;
     int j = 0;
     assert(memoTable != NULL);
     memset(memoTable, 0, arrayLen);
     paramConstraints = cParamUnsetMin(paramConstraints);
 
-
     for(i = 0; i < arrayLen; i++) {
         memoTableIndInv(¶mConstraints, varyParams, varyLen, i);
-        if(ZSTD_estimateCStreamSize_usingCParams(pvalsToCParams(paramConstraints)) > (size_t)target.cMem) {
-            memoTable[i] = 255;
-            j++;
-        }
-        if(wFixed && (1ULL << (paramConstraints.vals[wlog_ind] - 1)) >= srcSize && paramConstraints.vals[wlog_ind] != mintable[wlog_ind]) {
-            memoTable[i] = 255;
-        }
-        /* nil out parameter sets equivalent to others. */
-        if(cwFixed) {
-            if(paramConstraints.vals[strt_ind] == ZSTD_btlazy2 || paramConstraints.vals[strt_ind] == ZSTD_btopt || paramConstraints.vals[strt_ind] == ZSTD_btultra) {
-                if(paramConstraints.vals[clog_ind] > paramConstraints.vals[wlog_ind]+ 1) {
-                    if(memoTable[i] != 255) { j++; }
-                    memoTable[i] = 255;
-                }
-            } else {
-                if(paramConstraints.vals[clog_ind] > paramConstraints.vals[wlog_ind]) {
-                    if(memoTable[i] != 255) { j++; }
-                    memoTable[i] = 255;
-                }
-            }
-        }
-
-        if(scFixed) {
-            if(paramConstraints.vals[slog_ind] > paramConstraints.vals[clog_ind]) {
-                if(memoTable[i] != 255) { j++; }
-                memoTable[i] = 255;
-            }
-        }
-
-        if(whFixed) {
-            if(paramConstraints.vals[hlog_ind] > paramConstraints.vals[wlog_ind] + 1) {
-                if(memoTable[i] != 255) { j++; }
-                memoTable[i] = 255;
-            }
+        if(redundantParams(paramConstraints, target, srcSize)) {
+            memoTable[i] = 255; j++;
         }
     }
-    DEBUGOUTPUT("%d / %d Invalid\n", j, (int)i);
-    if((int)i == j) {
-        DEBUGOUTPUT("!!!Strategy %d totally infeasible\n", (int)paramConstraints.vals[strt_ind]);
-    }
+
+    DEBUGOUTPUT("%d / %d Invalid\n", j, (int)arrayLen);
 }
 
 /* frees all allocated memotables */
@@ -1539,7 +1599,7 @@ static void freeMemoTableArray(U8** mtAll) {
 
 /* inits memotables for all (including mallocs), all strategies */
 /* takes unsanitized varyParams */
-static U8** createMemoTableArray(paramValues_t paramConstraints, constraint_t target, const varInds_t* varyParams, const int varyLen, const size_t srcSize) {
+static U8** createMemoTableArray(paramValues_t paramConstraints, const constraint_t target, const varInds_t* varyParams, const int varyLen, const size_t srcSize) {
     varInds_t varNew[NUM_PARAMS];
     U8** mtAll = (U8**)calloc(sizeof(U8*),(ZSTD_btultra + 1));
     int i;
@@ -1549,18 +1609,21 @@ static U8** createMemoTableArray(paramValues_t paramConstraints, constraint_t ta
 
     for(i = 1; i <= (int)ZSTD_btultra; i++) {
         const int varLenNew = sanitizeVarArray(varNew, varyLen, varyParams, i);
-        mtAll[i] = malloc(sizeof(U8) * memoTableLen(varNew, varLenNew));
+        size_t mtl = memoTableLen(varNew, varLenNew);
+        if(mtl > g_memoLimit) { mtAll[i] = NULL; continue; }
+        mtAll[i] = malloc(sizeof(U8) * mtl);
         if(mtAll[i] == NULL) {
             freeMemoTableArray(mtAll);
             return NULL;
         }
+        paramConstraints.vals[strt_ind] = i;
         initMemoTable(mtAll[i], paramConstraints, target, varNew, varLenNew, srcSize);
     }
     
     return mtAll;
 }
 
-static paramValues_t overwriteParams(paramValues_t base, paramValues_t mask) {
+static paramValues_t overwriteParams(paramValues_t base, const paramValues_t mask) {
     U32 i;
     for(i = 0; i < NUM_PARAMS; i++) {
         if(mask.vals[i] != PARAM_UNSET) {
@@ -1586,7 +1649,7 @@ static BYTE* NB_TESTS_PLAYED(ZSTD_compressionParameters p) {
 
 static void playAround(FILE* f, oldWinnerInfo_t* winners,
                        ZSTD_compressionParameters params,
-                       buffers_t buf, contexts_t ctx)
+                       const buffers_t buf, const contexts_t ctx)
 {
     int nbVariations = 0;
     UTIL_time_t const clockStart = UTIL_getTime();
@@ -1619,49 +1682,35 @@ static void playAround(FILE* f, oldWinnerInfo_t* winners,
 
 }
 
+/* Sets pc to random unmeasured set of parameters */
+/* Doesn't do strategy! */
+static void randomConstrainedParams(paramValues_t* pc, const varInds_t* varArray, const int varLen, const U8* memoTable)
+{
+    size_t j;
+    for(j = 0; j < MIN(g_memoLimit, memoTableLen(varArray, varLen)); j++) {
+        int i;
+        for(i = 0; i < NUM_PARAMS; i++) {
+            varInds_t v = varArray[i];
+            if(v == strt_ind) continue; //don't do strategy (dependent on MT)
+            pc->vals[v] = rangeMap(v, FUZ_rand(&g_rand) % rangetable[v]);
+        }
+
+        if(memoTable == NULL || memoTable[memoTableInd(pc, varArray, varLen)]) { break; }
+    }
+}
+
 /* Completely random parameter selection */
 static ZSTD_compressionParameters randomParams(void)
 {
-    ZSTD_compressionParameters p;
-    U32 validated = 0;
-    while (!validated) {
-        /* totally random entry */
-        p.chainLog   = (FUZ_rand(&g_rand) % (ZSTD_CHAINLOG_MAX+1 - ZSTD_CHAINLOG_MIN))         
-            + ZSTD_CHAINLOG_MIN;
-        p.hashLog    = (FUZ_rand(&g_rand) % (ZSTD_HASHLOG_MAX+1 - ZSTD_HASHLOG_MIN))           
-            + ZSTD_HASHLOG_MIN;
-        p.searchLog  = (FUZ_rand(&g_rand) % (ZSTD_SEARCHLOG_MAX+1 - ZSTD_SEARCHLOG_MIN))       
-            + ZSTD_SEARCHLOG_MIN;
-        p.windowLog  = (FUZ_rand(&g_rand) % (ZSTD_WINDOWLOG_MAX+1 - ZSTD_WINDOWLOG_MIN))       
-            + ZSTD_WINDOWLOG_MIN;
-        p.searchLength=(FUZ_rand(&g_rand) % (ZSTD_SEARCHLENGTH_MAX+1 - ZSTD_SEARCHLENGTH_MIN)) 
-            + ZSTD_SEARCHLENGTH_MIN;
-        p.targetLength=(FUZ_rand(&g_rand) % (512));
-
-        p.strategy   = (ZSTD_strategy) (FUZ_rand(&g_rand) % (ZSTD_btultra +1));
-        
-        validated = !ZSTD_isError(ZSTD_checkCParams(p));
-    }
-    return p;
-}
-
-/* Sets pc to random unmeasured set of parameters */
-static void randomConstrainedParams(paramValues_t* pc, varInds_t* varArray, int varLen, U8* memoTable)
-{
-    size_t tries = memoTableLen(varArray, varLen); 
-    const size_t maxSize = memoTableLen(varArray, varLen);
-    size_t ind;
-    do {
-        ind = (FUZ_rand(&g_rand)) % maxSize;
-        tries--;
-    } while(memoTable[ind] > 0 && tries > 0); 
-
-    memoTableIndInv(pc, varArray, varLen, (unsigned)ind);
+    paramValues_t p;
+    p.vals[strt_ind] = rangeMap(strt_ind, rangeMap(strt_ind, FUZ_rand(&g_rand) % rangetable[strt_ind]));
+    randomConstrainedParams(&p, paramTable, NUM_PARAMS, NULL);
+    return pvalsToCParams(p);
 }
 
 static void BMK_selectRandomStart(
                        FILE* f, oldWinnerInfo_t* winners,
-                       buffers_t buf, contexts_t ctx)
+                       const buffers_t buf, const contexts_t ctx)
 {
     U32 const id = FUZ_rand(&g_rand) % (NB_LEVELS_TRACKED+1);
     if ((id==0) || (winners[id].params.windowLog==0)) {
@@ -1673,7 +1722,7 @@ static void BMK_selectRandomStart(
     }
 }
 
-static void BMK_benchFullTable(buffers_t buf, contexts_t ctx, const size_t maxBlockSize) 
+static void BMK_benchFullTable(const buffers_t buf, const contexts_t ctx) 
 {
     ZSTD_compressionParameters params;
     oldWinnerInfo_t winners[NB_LEVELS_TRACKED+1];
@@ -1689,7 +1738,7 @@ static void BMK_benchFullTable(buffers_t buf, contexts_t ctx, const size_t maxBl
         BMK_init_level_constraints(g_target * (1 MB));
     } else {
         /* baseline config for level 1 */
-        ZSTD_compressionParameters const l1params = ZSTD_getCParams(1, maxBlockSize, ctx.dictSize);
+        ZSTD_compressionParameters const l1params = ZSTD_getCParams(1, buf.maxBlockSize, ctx.dictSize);
         BMK_result_t testResult;
         BMK_benchParam(&testResult, buf, ctx, cParamsToPVals(l1params));
         BMK_init_level_constraints((int)((testResult.cSpeed * 31) / 32));
@@ -1699,7 +1748,7 @@ static void BMK_benchFullTable(buffers_t buf, contexts_t ctx, const size_t maxBl
     {   const int maxSeeds = g_noSeed ? 1 : ZSTD_maxCLevel();
         int i;
         for (i=0; i<=maxSeeds; i++) {
-            params = ZSTD_getCParams(i, maxBlockSize, 0);
+            params = ZSTD_getCParams(i, buf.maxBlockSize, 0);
             BMK_seed(winners, params, buf, ctx);
     }   }
     BMK_printWinners(f, winners, buf.srcSize);
@@ -1719,7 +1768,7 @@ static void BMK_benchFullTable(buffers_t buf, contexts_t ctx, const size_t maxBl
     fclose(f);
 }
 
-static int benchOnce(buffers_t buf, contexts_t ctx) {
+static int benchOnce(const buffers_t buf, const contexts_t ctx) {
     BMK_result_t testResult;
 
     if(BMK_benchParam(&testResult, buf, ctx, g_params)) {
@@ -1736,7 +1785,6 @@ static int benchSample(void)
 {
     const char* const name = "Sample 10MB";
     size_t const benchedSize = 10 MB;
-    U32 blockSize = g_blockSize ? g_blockSize : benchedSize;
     void* srcBuffer = malloc(benchedSize);
     int ret = 0;
 
@@ -1769,7 +1817,7 @@ static int benchSample(void)
     if(g_singleRun) {
         ret = benchOnce(buf, ctx);
     } else {
-        BMK_benchFullTable(buf, ctx, MIN(blockSize, benchedSize));
+        BMK_benchFullTable(buf, ctx);
     }
 
     freeBuffers(buf);
@@ -1785,7 +1833,6 @@ int benchFiles(const char** fileNamesTable, int nbFiles, const char* dictFileNam
 {
     buffers_t buf;
     contexts_t ctx;
-    size_t maxBlockSize = 0, i;
     int ret = 0;
 
     if(createBuffers(&buf, fileNamesTable, nbFiles)) {
@@ -1799,10 +1846,6 @@ int benchFiles(const char** fileNamesTable, int nbFiles, const char* dictFileNam
         return 2;
     }
 
-    for(i = 0; i < buf.nbBlocks; i++) {
-        maxBlockSize = MAX(maxBlockSize, buf.srcSizes[i]);
-    }
-
     DISPLAY("\r%79s\r", "");
     if(nbFiles == 1) {
         DISPLAY("using %s : \n", fileNamesTable[0]);
@@ -1810,12 +1853,12 @@ int benchFiles(const char** fileNamesTable, int nbFiles, const char* dictFileNam
         DISPLAY("using %d Files : \n", nbFiles);
     }
 
-    g_params = adjustParams(overwriteParams(cParamsToPVals(ZSTD_getCParams(cLevel, maxBlockSize, ctx.dictSize)), g_params), maxBlockSize, ctx.dictSize);
+    g_params = adjustParams(overwriteParams(cParamsToPVals(ZSTD_getCParams(cLevel, buf.maxBlockSize, ctx.dictSize)), g_params), buf.maxBlockSize, ctx.dictSize);
 
     if(g_singleRun) {
         ret = benchOnce(buf, ctx);
     } else {
-        BMK_benchFullTable(buf, ctx, maxBlockSize);
+        BMK_benchFullTable(buf, ctx);
     }
 
     freeBuffers(buf);
@@ -1937,7 +1980,7 @@ static int benchMemo(BMK_result_t* resultPtr,
     size_t memind = memoTableInd(&cParams, varyParams, varyLen);
     int res;
 
-    if(memoTable[memind] >= INFEASIBLE_THRESHOLD) { return WORSE_RESULT; } 
+    if((memoTable == NULL && redundantParams(cParams, target, buf.maxBlockSize)) || ((memoTable != NULL) && memoTable[memind] >= INFEASIBLE_THRESHOLD)) { return WORSE_RESULT; } 
 
     res = allBench(resultPtr, buf, ctx, cParams, target, winnerResult, feas);
 
@@ -1947,7 +1990,7 @@ static int benchMemo(BMK_result_t* resultPtr,
     }
     BMK_printWinnerOpt(stdout, CUSTOM_LEVEL, *resultPtr, cParams, target, buf.srcSize);
 
-    if(res == BETTER_RESULT || feas) {
+    if(memoTable != NULL && (res == BETTER_RESULT || feas)) {
         memoTable[memind] = 255; 
     }
     return res;
@@ -1972,7 +2015,7 @@ static int benchMemo(BMK_result_t* resultPtr,
 static winnerInfo_t climbOnce(const constraint_t target, 
                 const varInds_t* varArray, const int varLen, ZSTD_strategy strat,
                 U8** memoTableArray, 
-                buffers_t buf, contexts_t ctx,
+                const buffers_t buf, const contexts_t ctx,
                 const paramValues_t init) {
     /* 
      * cparam - currently considered 'center'
@@ -2103,7 +2146,6 @@ static winnerInfo_t optimizeFixedStrategy(
     /* so climb is given the right fixed strategy */
     paramTarget.vals[strt_ind] = strat;
     /* to pass ZSTD_checkCParams */
-
     paramTarget = cParamUnsetMin(paramTarget);
 
     init = paramTarget;
@@ -2178,16 +2220,11 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_
     const int varLen = variableParams(paramTarget, varArray, dictFileName != NULL);
     winnerInfo_t winner = initWinnerInfo(emptyParams());
     U8** allMT = NULL;
-    paramValues_t paramBase = cParamUnsetMin(paramTarget);
-    size_t k, maxBlockSize = 0;
+    paramValues_t paramBase;
     contexts_t ctx;
     buffers_t buf;
     g_time = UTIL_getTime();
 
-    if(!paramValid(paramBase)) {
-        return 1;
-    }
-
     if(createBuffers(&buf, fileNamesTable, nbFiles)) {
         DISPLAY("unable to load files\n");
         return 1;
@@ -2205,10 +2242,9 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_
         DISPLAY("Loading %lu Files...       \r", (unsigned long)nbFiles); 
     }
 
-
-    for(k = 0; k < buf.nbBlocks; k++) {
-        maxBlockSize = MAX(buf.srcSizes[k], maxBlockSize);
-    }
+    /* sanitize paramTarget */
+    optimizerAdjustInput(¶mTarget, buf.maxBlockSize);
+    paramBase = cParamUnsetMin(paramTarget);
 
     /* if strategy is fixed, only init that part of memotable */
     if(paramTarget.vals[strt_ind] != PARAM_UNSET) {
@@ -2227,9 +2263,9 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_
             goto _cleanUp;
         }
         
-        initMemoTable(allMT[paramTarget.vals[strt_ind]], paramTarget, target, varNew, varLenNew, maxBlockSize);
+        initMemoTable(allMT[paramTarget.vals[strt_ind]], paramTarget, target, varNew, varLenNew, buf.maxBlockSize);
     } else {
-         allMT = createMemoTableArray(paramTarget, target, varArray, varLen, maxBlockSize);
+         allMT = createMemoTableArray(paramTarget, target, varArray, varLen, buf.maxBlockSize);
     }
    
     if(!allMT) {
@@ -2256,7 +2292,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_
     /* use level'ing mode instead of normal target mode */
     /* Should lvl be parameter-masked here? */
     if(g_optmode) {
-        winner.params = cParamsToPVals(ZSTD_getCParams(cLevelOpt, maxBlockSize, ctx.dictSize));
+        winner.params = cParamsToPVals(ZSTD_getCParams(cLevelOpt, buf.maxBlockSize, ctx.dictSize));
         if(BMK_benchParam(&winner.result, buf, ctx, winner.params)) {
             ret = 3;
             goto _cleanUp;
@@ -2276,7 +2312,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_
     /* Don't want it to return anything worse than the best known result */
     if(g_singleRun) {
         BMK_result_t res;
-        g_params = adjustParams(overwriteParams(cParamsToPVals(ZSTD_getCParams(cLevelRun, maxBlockSize, ctx.dictSize)), g_params), maxBlockSize, ctx.dictSize);
+        g_params = adjustParams(overwriteParams(cParamsToPVals(ZSTD_getCParams(cLevelRun, buf.maxBlockSize, ctx.dictSize)), g_params), buf.maxBlockSize, ctx.dictSize);
         if(BMK_benchParam(&res, buf, ctx, g_params)) {
             ret = 45;
             goto _cleanUp;
@@ -2315,7 +2351,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_
                 int i;
                 for (i=1; i<=maxSeeds; i++) {
                     int ec;
-                    CParams = overwriteParams(cParamsToPVals(ZSTD_getCParams(i, maxBlockSize, ctx.dictSize)), paramTarget);
+                    CParams = overwriteParams(cParamsToPVals(ZSTD_getCParams(i, buf.maxBlockSize, ctx.dictSize)), paramTarget);
                     ec = BMK_benchParam(&candidate, buf, ctx, CParams);
                     BMK_printWinnerOpt(stdout, i, candidate, CParams, target, buf.srcSize);
 
@@ -2466,16 +2502,24 @@ static int badusage(const char* exename)
 }
 
 #define PARSE_SUB_ARGS(stringLong, stringShort, variable) { if (longCommandWArg(&argument, stringLong) || longCommandWArg(&argument, stringShort)) { variable = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; } }
-#define PARSE_CPARAMS(variable)                                            \
-{                                                                          \
-    PARSE_SUB_ARGS("windowLog=",       "wlog=",  variable.vals[wlog_ind]); \
-    PARSE_SUB_ARGS("chainLog=" ,       "clog=",  variable.vals[clog_ind]); \
-    PARSE_SUB_ARGS("hashLog=",         "hlog=",  variable.vals[hlog_ind]); \
-    PARSE_SUB_ARGS("searchLog=" ,      "slog=",  variable.vals[slog_ind]); \
-    PARSE_SUB_ARGS("searchLength=",    "slen=",  variable.vals[slen_ind]); \
-    PARSE_SUB_ARGS("targetLength=" ,   "tlen=",  variable.vals[tlen_ind]); \
-    PARSE_SUB_ARGS("strategy=",        "strat=", variable.vals[strt_ind]); \
-    PARSE_SUB_ARGS("forceAttachDict=", "fad="  , variable.vals[fadt_ind]); \
+/* 1 if successful parse, 0 otherwise */
+static int parse_params(const char** argptr, paramValues_t* pv) {
+    int matched = 0;
+    const char* argOrig = *argptr;
+    varInds_t v;
+    for(v = 0; v < NUM_PARAMS; v++) {
+        if(longCommandWArg(argptr,g_shortParamNames[v]) || longCommandWArg(argptr, g_paramNames[v])) {
+            if(**argptr == '=') {
+                (*argptr)++;
+                pv->vals[v] = readU32FromChar(argptr);
+                matched = 1;
+                break;
+            }
+        }
+        /* reset and try again */
+        *argptr = argOrig;
+    }
+    return matched;
 }
 
 int main(int argc, const char** argv)
@@ -2509,7 +2553,7 @@ int main(int argc, const char** argv)
         if (longCommandWArg(&argument, "--optimize=")) {
             g_optimizer = 1;
             for ( ; ;) {
-                PARSE_CPARAMS(paramTarget);
+                if(parse_params(&argument, ¶mTarget)) { if(argument[0] == ',') { argument++; continue; } else break; }
                 PARSE_SUB_ARGS("compressionSpeed=" ,  "cSpeed=", target.cSpeed);
                 PARSE_SUB_ARGS("decompressionSpeed=", "dSpeed=", target.dSpeed); 
                 PARSE_SUB_ARGS("compressionMemory=" , "cMem=", target.cMem);
@@ -2517,6 +2561,7 @@ int main(int argc, const char** argv)
                 PARSE_SUB_ARGS("preferSpeed=", "prfSpd=", g_speedMultiplier);
                 PARSE_SUB_ARGS("preferRatio=", "prfRto=", g_ratioMultiplier);
                 PARSE_SUB_ARGS("maxTries=", "tries=", g_maxTries);
+                PARSE_SUB_ARGS("memoLimit=", "memo=", g_memoLimit);
                 if (longCommandWArg(&argument, "level=") || longCommandWArg(&argument, "lvl=")) { cLevelOpt = readU32FromChar(&argument); g_optmode = 1; if (argument[0]==',') { argument++; continue; } else break; }
 
                 DISPLAY("invalid optimization parameter \n");
@@ -2532,7 +2577,7 @@ int main(int argc, const char** argv)
         /* Decode command (note : aggregated commands are allowed) */
             g_singleRun = 1;
             for ( ; ;) {
-                PARSE_CPARAMS(g_params)
+                if(parse_params(&argument, &g_params)) { if(argument[0] == ',') { argument++; continue; } else break; }
                 if (longCommandWArg(&argument, "level=") || longCommandWArg(&argument, "lvl=")) { cLevelRun = readU32FromChar(&argument); g_params = emptyParams(); if (argument[0]==',') { argument++; continue; } else break; }
 
                 DISPLAY("invalid compression parameter \n");

From 6e66bbf5dde09f555a2adb1dd763808c0de3c6c6 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Tue, 14 Aug 2018 12:56:21 -0700
Subject: [PATCH 166/372] fixed several minor issues detected by scan-build

only notable one :
writeNCount() resists better vs invalid distributions
(though it should never happen within zstd anyway)
---
 lib/common/zstd_internal.h       |  3 +-
 lib/compress/fse_compress.c      | 65 ++++++++++++++++++--------------
 lib/compress/zstd_compress.c     |  1 -
 lib/compress/zstd_opt.c          |  2 +-
 lib/decompress/zstd_decompress.c | 26 ++++++-------
 lib/dictBuilder/zdict.c          |  2 +-
 lib/legacy/zstd_v04.c            |  1 +
 lib/legacy/zstd_v05.c            |  1 +
 lib/legacy/zstd_v06.c            |  2 +-
 lib/legacy/zstd_v07.c            | 20 +++++-----
 10 files changed, 67 insertions(+), 56 deletions(-)

diff --git a/lib/common/zstd_internal.h b/lib/common/zstd_internal.h
index b4c1af53f..b36a2fbd0 100644
--- a/lib/common/zstd_internal.h
+++ b/lib/common/zstd_internal.h
@@ -79,8 +79,7 @@ static const U32 repStartValue[ZSTD_REP_NUM] = { 1, 4, 8 };
 static const size_t ZSTD_fcs_fieldSize[4] = { 0, 2, 4, 8 };
 static const size_t ZSTD_did_fieldSize[4] = { 0, 1, 2, 4 };
 
-#define ZSTD_FRAMEIDSIZE 4
-static const size_t ZSTD_frameIdSize = ZSTD_FRAMEIDSIZE;  /* magic number size */
+#define ZSTD_FRAMEIDSIZE 4   /* magic number size */
 
 #define ZSTD_BLOCKHEADERSIZE 3   /* C standard doesn't allow `static const` variable to be init using another `static const` variable */
 static const size_t ZSTD_blockHeaderSize = ZSTD_BLOCKHEADERSIZE;
diff --git a/lib/compress/fse_compress.c b/lib/compress/fse_compress.c
index 70daae3bc..caa90edec 100644
--- a/lib/compress/fse_compress.c
+++ b/lib/compress/fse_compress.c
@@ -83,7 +83,9 @@
  * wkspSize should be sized to handle worst case situation, which is `1< wkspSize) return ERROR(tableLog_tooLarge);
     tableU16[-2] = (U16) tableLog;
     tableU16[-1] = (U16) maxSymbolValue;
-    assert(tableLog < 16);   /* required for the threshold strategy to work */
+    assert(tableLog < 16);   /* required for threshold strategy to work */
 
     /* For explanations on how to distribute symbol values over the table :
-    *  http://fastcompression.blogspot.fr/2014/02/fse-distributing-symbol-values.html */
+     * http://fastcompression.blogspot.fr/2014/02/fse-distributing-symbol-values.html */
+
+     #ifdef __clang_analyzer__
+     memset(tableSymbol, 0, sizeof(*tableSymbol) * tableSize);   /* useless initialization, just to keep scan-build happy */
+     #endif
 
     /* symbol start positions */
     {   U32 u;
@@ -124,13 +130,15 @@ size_t FSE_buildCTable_wksp(FSE_CTable* ct, const short* normalizedCounter, unsi
         U32 symbol;
         for (symbol=0; symbol<=maxSymbolValue; symbol++) {
             int nbOccurences;
-            for (nbOccurences=0; nbOccurences highThreshold) position = (position + step) & tableMask;   /* Low proba area */
+                while (position > highThreshold)
+                    position = (position + step) & tableMask;   /* Low proba area */
         }   }
 
-        if (position!=0) return ERROR(GENERIC);   /* Must have gone through all positions */
+        assert(position==0);  /* Must have initialized all positions */
     }
 
     /* Build table */
@@ -201,9 +209,10 @@ size_t FSE_NCountWriteBound(unsigned maxSymbolValue, unsigned tableLog)
     return maxSymbolValue ? maxHeaderSize : FSE_NCOUNTBOUND;  /* maxSymbolValue==0 ? use default */
 }
 
-static size_t FSE_writeNCount_generic (void* header, size_t headerBufferSize,
-                                       const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog,
-                                       unsigned writeIsSafe)
+static size_t
+FSE_writeNCount_generic (void* header, size_t headerBufferSize,
+                   const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog,
+                         unsigned writeIsSafe)
 {
     BYTE* const ostart = (BYTE*) header;
     BYTE* out = ostart;
@@ -212,13 +221,12 @@ static size_t FSE_writeNCount_generic (void* header, size_t headerBufferSize,
     const int tableSize = 1 << tableLog;
     int remaining;
     int threshold;
-    U32 bitStream;
-    int bitCount;
-    unsigned charnum = 0;
-    int previous0 = 0;
+    U32 bitStream = 0;
+    int bitCount = 0;
+    unsigned symbol = 0;
+    unsigned const alphabetSize = maxSymbolValue + 1;
+    int previousIs0 = 0;
 
-    bitStream = 0;
-    bitCount  = 0;
     /* Table Size */
     bitStream += (tableLog-FSE_MIN_TABLELOG) << bitCount;
     bitCount  += 4;
@@ -228,11 +236,11 @@ static size_t FSE_writeNCount_generic (void* header, size_t headerBufferSize,
     threshold = tableSize;
     nbBits = tableLog+1;
 
-    while (remaining>1) {  /* stops at 1 */
-        if (previous0) {
-            unsigned start = charnum;
-            while (!normalizedCounter[charnum]) charnum++;
-            while (charnum >= start+24) {
+    while ((symbol < alphabetSize) && (remaining>1)) {  /* stops at 1 */
+        if (previousIs0) {
+            unsigned start = symbol;
+            while ((symbol < alphabetSize) && !normalizedCounter[symbol]) symbol++;
+            while (symbol >= start+24) {
                 start+=24;
                 bitStream += 0xFFFFU << bitCount;
                 if ((!writeIsSafe) && (out > oend-2)) return ERROR(dstSize_tooSmall);   /* Buffer overflow */
@@ -241,12 +249,12 @@ static size_t FSE_writeNCount_generic (void* header, size_t headerBufferSize,
                 out+=2;
                 bitStream>>=16;
             }
-            while (charnum >= start+3) {
+            while (symbol >= start+3) {
                 start+=3;
                 bitStream += 3 << bitCount;
                 bitCount += 2;
             }
-            bitStream += (charnum-start) << bitCount;
+            bitStream += (symbol-start) << bitCount;
             bitCount += 2;
             if (bitCount>16) {
                 if ((!writeIsSafe) && (out > oend - 2)) return ERROR(dstSize_tooSmall);   /* Buffer overflow */
@@ -256,15 +264,15 @@ static size_t FSE_writeNCount_generic (void* header, size_t headerBufferSize,
                 bitStream >>= 16;
                 bitCount -= 16;
         }   }
-        {   int count = normalizedCounter[charnum++];
-            int const max = (2*threshold-1)-remaining;
+        {   int count = normalizedCounter[symbol++];
+            int const max = (2*threshold-1) - remaining;
             remaining -= count < 0 ? -count : count;
             count++;   /* +1 for extra accuracy */
             if (count>=threshold) count += max;   /* [0..max[ [max..threshold[ (...) [threshold+max 2*threshold[ */
             bitStream += count << bitCount;
             bitCount  += nbBits;
             bitCount  -= (count>=1; }
         }
@@ -277,14 +285,15 @@ static size_t FSE_writeNCount_generic (void* header, size_t headerBufferSize,
             bitCount -= 16;
     }   }
 
+    if (remaining != 1) return ERROR(GENERIC);  /* incorrect normalized distribution */
+    assert(symbol <= alphabetSize);
+
     /* flush remaining bitStream */
     if ((!writeIsSafe) && (out > oend - 2)) return ERROR(dstSize_tooSmall);   /* Buffer overflow */
     out[0] = (BYTE)bitStream;
     out[1] = (BYTE)(bitStream>>8);
     out+= (bitCount+7) /8;
 
-    if (charnum > maxSymbolValue + 1) return ERROR(GENERIC);
-
     return (out-ostart);
 }
 
@@ -297,7 +306,7 @@ size_t FSE_writeNCount (void* buffer, size_t bufferSize, const short* normalized
     if (bufferSize < FSE_NCountWriteBound(maxSymbolValue, tableLog))
         return FSE_writeNCount_generic(buffer, bufferSize, normalizedCounter, maxSymbolValue, tableLog, 0);
 
-    return FSE_writeNCount_generic(buffer, bufferSize, normalizedCounter, maxSymbolValue, tableLog, 1);
+    return FSE_writeNCount_generic(buffer, bufferSize, normalizedCounter, maxSymbolValue, tableLog, 1 /* write in buffer is safe */);
 }
 
 
diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index ed3aab871..29dce1250 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -1147,7 +1147,6 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc,
                 if (zc->workSpace == NULL) return ERROR(memory_allocation);
                 zc->workSpaceSize = neededSpace;
                 zc->workSpaceOversizedDuration = 0;
-                ptr = zc->workSpace;
 
                 /* Statically sized space.
                  * entropyWorkspace never moves,
diff --git a/lib/compress/zstd_opt.c b/lib/compress/zstd_opt.c
index 476cdc148..c4b9bb13b 100644
--- a/lib/compress/zstd_opt.c
+++ b/lib/compress/zstd_opt.c
@@ -970,7 +970,7 @@ _shortestPath:   /* cur, last_pos, best_mlen, best_off have to be set */
             U32 seqPos = cur;
 
             DEBUGLOG(6, "start reverse traversal (last_pos:%u, cur:%u)",
-                        last_pos, cur);
+                        last_pos, cur); (void)last_pos;
             assert(storeEnd < ZSTD_OPT_NUM);
             DEBUGLOG(6, "last sequence copied into pos=%u (llen=%u,mlen=%u,ofc=%u)",
                         storeEnd, lastSequence.litlen, lastSequence.mlen, lastSequence.off);
diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c
index 8f4589d13..16a08fb6b 100644
--- a/lib/decompress/zstd_decompress.c
+++ b/lib/decompress/zstd_decompress.c
@@ -185,7 +185,7 @@ size_t ZSTD_estimateDCtxSize(void) { return sizeof(ZSTD_DCtx); }
 static size_t ZSTD_startingInputLength(ZSTD_format_e format)
 {
     size_t const startingInputLength = (format==ZSTD_f_zstd1_magicless) ?
-                    ZSTD_frameHeaderSize_prefix - ZSTD_frameIdSize :
+                    ZSTD_frameHeaderSize_prefix - ZSTD_FRAMEIDSIZE :
                     ZSTD_frameHeaderSize_prefix;
     ZSTD_STATIC_ASSERT(ZSTD_FRAMEHEADERSIZE_PREFIX >= ZSTD_FRAMEIDSIZE);
     /* only supports formats ZSTD_f_zstd1 and ZSTD_f_zstd1_magicless */
@@ -278,7 +278,7 @@ void ZSTD_copyDCtx(ZSTD_DCtx* dstDCtx, const ZSTD_DCtx* srcDCtx)
  *  Note 3 : Skippable Frame Identifiers are considered valid. */
 unsigned ZSTD_isFrame(const void* buffer, size_t size)
 {
-    if (size < ZSTD_frameIdSize) return 0;
+    if (size < ZSTD_FRAMEIDSIZE) return 0;
     {   U32 const magic = MEM_readLE32(buffer);
         if (magic == ZSTD_MAGICNUMBER) return 1;
         if ((magic & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) return 1;
@@ -331,6 +331,7 @@ size_t ZSTD_getFrameHeader_advanced(ZSTD_frameHeader* zfhPtr, const void* src, s
     size_t const minInputSize = ZSTD_startingInputLength(format);
 
     if (srcSize < minInputSize) return minInputSize;
+    if (src==NULL) return ERROR(GENERIC);   /* invalid parameter */
 
     if ( (format != ZSTD_f_zstd1_magicless)
       && (MEM_readLE32(src) != ZSTD_MAGICNUMBER) ) {
@@ -339,7 +340,7 @@ size_t ZSTD_getFrameHeader_advanced(ZSTD_frameHeader* zfhPtr, const void* src, s
             if (srcSize < ZSTD_skippableHeaderSize)
                 return ZSTD_skippableHeaderSize; /* magic number + frame length */
             memset(zfhPtr, 0, sizeof(*zfhPtr));
-            zfhPtr->frameContentSize = MEM_readLE32((const char *)src + ZSTD_frameIdSize);
+            zfhPtr->frameContentSize = MEM_readLE32((const char *)src + ZSTD_FRAMEIDSIZE);
             zfhPtr->frameType = ZSTD_skippableFrame;
             return 0;
         }
@@ -451,7 +452,7 @@ unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize)
             size_t skippableSize;
             if (srcSize < ZSTD_skippableHeaderSize)
                 return ERROR(srcSize_wrong);
-            skippableSize = MEM_readLE32((const BYTE *)src + ZSTD_frameIdSize)
+            skippableSize = MEM_readLE32((const BYTE *)src + ZSTD_FRAMEIDSIZE)
                           + ZSTD_skippableHeaderSize;
             if (srcSize < skippableSize) {
                 return ZSTD_CONTENTSIZE_ERROR;
@@ -1763,7 +1764,7 @@ size_t ZSTD_findFrameCompressedSize(const void *src, size_t srcSize)
 #endif
     if ( (srcSize >= ZSTD_skippableHeaderSize)
       && (MEM_readLE32(src) & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START ) {
-        return ZSTD_skippableHeaderSize + MEM_readLE32((const BYTE*)src + ZSTD_frameIdSize);
+        return ZSTD_skippableHeaderSize + MEM_readLE32((const BYTE*)src + ZSTD_FRAMEIDSIZE);
     } else {
         const BYTE* ip = (const BYTE*)src;
         const BYTE* const ipstart = ip;
@@ -1797,7 +1798,6 @@ size_t ZSTD_findFrameCompressedSize(const void *src, size_t srcSize)
         if (zfh.checksumFlag) {   /* Final frame content checksum */
             if (remainingSize < 4) return ERROR(srcSize_wrong);
             ip += 4;
-            remainingSize -= 4;
         }
 
         return ip - ipstart;
@@ -1932,7 +1932,7 @@ static size_t ZSTD_decompressMultiFrame(ZSTD_DCtx* dctx,
                 size_t skippableSize;
                 if (srcSize < ZSTD_skippableHeaderSize)
                     return ERROR(srcSize_wrong);
-                skippableSize = MEM_readLE32((const BYTE*)src + ZSTD_frameIdSize)
+                skippableSize = MEM_readLE32((const BYTE*)src + ZSTD_FRAMEIDSIZE)
                               + ZSTD_skippableHeaderSize;
                 if (srcSize < skippableSize) return ERROR(srcSize_wrong);
 
@@ -2057,7 +2057,7 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, c
     case ZSTDds_getFrameHeaderSize :
         assert(src != NULL);
         if (dctx->format == ZSTD_f_zstd1) {  /* allows header */
-            assert(srcSize >= ZSTD_frameIdSize);  /* to read skippable magic number */
+            assert(srcSize >= ZSTD_FRAMEIDSIZE);  /* to read skippable magic number */
             if ((MEM_readLE32(src) & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) {        /* skippable frame */
                 memcpy(dctx->headerBuffer, src, srcSize);
                 dctx->expected = ZSTD_skippableHeaderSize - srcSize;  /* remaining to load to get full skippable frame header */
@@ -2167,7 +2167,7 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, c
         assert(src != NULL);
         assert(srcSize <= ZSTD_skippableHeaderSize);
         memcpy(dctx->headerBuffer + (ZSTD_skippableHeaderSize - srcSize), src, srcSize);   /* complete skippable header */
-        dctx->expected = MEM_readLE32(dctx->headerBuffer + ZSTD_frameIdSize);   /* note : dctx->expected can grow seriously large, beyond local buffer size */
+        dctx->expected = MEM_readLE32(dctx->headerBuffer + ZSTD_FRAMEIDSIZE);   /* note : dctx->expected can grow seriously large, beyond local buffer size */
         dctx->stage = ZSTDds_skipFrame;
         return 0;
 
@@ -2268,7 +2268,7 @@ static size_t ZSTD_decompress_insertDictionary(ZSTD_DCtx* dctx, const void* dict
         if (magic != ZSTD_MAGIC_DICTIONARY) {
             return ZSTD_refDictContent(dctx, dict, dictSize);   /* pure content mode */
     }   }
-    dctx->dictID = MEM_readLE32((const char*)dict + ZSTD_frameIdSize);
+    dctx->dictID = MEM_readLE32((const char*)dict + ZSTD_FRAMEIDSIZE);
 
     /* load entropy tables */
     {   size_t const eSize = ZSTD_loadEntropy(&dctx->entropy, dict, dictSize);
@@ -2381,7 +2381,7 @@ static size_t ZSTD_loadEntropy_inDDict(ZSTD_DDict* ddict, ZSTD_dictContentType_e
             return 0;   /* pure content mode */
         }
     }
-    ddict->dictID = MEM_readLE32((const char*)ddict->dictContent + ZSTD_frameIdSize);
+    ddict->dictID = MEM_readLE32((const char*)ddict->dictContent + ZSTD_FRAMEIDSIZE);
 
     /* load entropy tables */
     CHECK_E( ZSTD_loadEntropy(&ddict->entropy, ddict->dictContent, ddict->dictSize), dictionary_corrupted );
@@ -2510,7 +2510,7 @@ unsigned ZSTD_getDictID_fromDict(const void* dict, size_t dictSize)
 {
     if (dictSize < 8) return 0;
     if (MEM_readLE32(dict) != ZSTD_MAGIC_DICTIONARY) return 0;
-    return MEM_readLE32((const char*)dict + ZSTD_frameIdSize);
+    return MEM_readLE32((const char*)dict + ZSTD_FRAMEIDSIZE);
 }
 
 /*! ZSTD_getDictID_fromDDict() :
@@ -2855,7 +2855,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
             CHECK_F(ZSTD_decompressBegin_usingDDict(zds, zds->ddict));
 
             if ((MEM_readLE32(zds->headerBuffer) & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) {  /* skippable frame */
-                zds->expected = MEM_readLE32(zds->headerBuffer + ZSTD_frameIdSize);
+                zds->expected = MEM_readLE32(zds->headerBuffer + ZSTD_FRAMEIDSIZE);
                 zds->stage = ZSTDds_skipFrame;
             } else {
                 CHECK_F(ZSTD_decodeFrameHeader(zds, zds->headerBuffer, zds->lhSize));
diff --git a/lib/dictBuilder/zdict.c b/lib/dictBuilder/zdict.c
index b8a51789a..a4d0a4481 100644
--- a/lib/dictBuilder/zdict.c
+++ b/lib/dictBuilder/zdict.c
@@ -698,7 +698,7 @@ static size_t ZDICT_analyzeEntropy(void*  dstBuffer, size_t maxDstSize,
     short litLengthNCount[MaxLL+1];
     U32 repOffset[MAXREPOFFSET];
     offsetCount_t bestRepOffset[ZSTD_REP_NUM+1];
-    EStats_ress_t esr;
+    EStats_ress_t esr = { NULL, NULL, NULL };
     ZSTD_parameters params;
     U32 u, huffLog = 11, Offlog = OffFSELog, mlLog = MLFSELog, llLog = LLFSELog, total;
     size_t pos = 0, errorCode;
diff --git a/lib/legacy/zstd_v04.c b/lib/legacy/zstd_v04.c
index a2e2cfa80..15000db6d 100644
--- a/lib/legacy/zstd_v04.c
+++ b/lib/legacy/zstd_v04.c
@@ -1093,6 +1093,7 @@ static size_t FSE_buildDTable(FSE_DTable* dt, const short* normalizedCounter, un
     if (tableLog > FSE_MAX_TABLELOG) return ERROR(tableLog_tooLarge);
 
     /* Init, lay down lowprob symbols */
+    memset(tableDecode, 0, sizeof(FSE_DECODE_TYPE) * (maxSymbolValue+1) );   /* useless init, but keep static analyzer happy, and we don't need to performance optimize legacy decoders */
     DTableH.tableLog = (U16)tableLog;
     for (s=0; s<=maxSymbolValue; s++)
     {
diff --git a/lib/legacy/zstd_v05.c b/lib/legacy/zstd_v05.c
index a5e1b1ffc..89a5fe7e6 100644
--- a/lib/legacy/zstd_v05.c
+++ b/lib/legacy/zstd_v05.c
@@ -1224,6 +1224,7 @@ size_t FSEv05_buildDTable(FSEv05_DTable* dt, const short* normalizedCounter, uns
     if (tableLog > FSEv05_MAX_TABLELOG) return ERROR(tableLog_tooLarge);
 
     /* Init, lay down lowprob symbols */
+    memset(tableDecode, 0, sizeof(FSEv05_FUNCTION_TYPE) * (maxSymbolValue+1) );   /* useless init, but keep static analyzer happy, and we don't need to performance optimize legacy decoders */
     DTableH.tableLog = (U16)tableLog;
     for (s=0; s<=maxSymbolValue; s++) {
         if (normalizedCounter[s]==-1) {
diff --git a/lib/legacy/zstd_v06.c b/lib/legacy/zstd_v06.c
index 8b068b3e5..d8de77ca9 100644
--- a/lib/legacy/zstd_v06.c
+++ b/lib/legacy/zstd_v06.c
@@ -4006,7 +4006,7 @@ size_t ZBUFFv06_decompressContinue(ZBUFFv06_DCtx* zbd,
                     if (ZSTDv06_isError(hSize)) return hSize;
                     if (toLoad > (size_t)(iend-ip)) {   /* not enough input to load full header */
                         memcpy(zbd->headerBuffer + zbd->lhSize, ip, iend-ip);
-                        zbd->lhSize += iend-ip; ip = iend; notDone = 0;
+                        zbd->lhSize += iend-ip;
                         *dstCapacityPtr = 0;
                         return (hSize - zbd->lhSize) + ZSTDv06_blockHeaderSize;   /* remaining header bytes + next block header */
                     }
diff --git a/lib/legacy/zstd_v07.c b/lib/legacy/zstd_v07.c
index 70b170f0f..9f95f62ad 100644
--- a/lib/legacy/zstd_v07.c
+++ b/lib/legacy/zstd_v07.c
@@ -3150,10 +3150,10 @@ size_t ZSTDv07_getFrameParams(ZSTDv07_frameParams* fparamsPtr, const void* src,
     const BYTE* ip = (const BYTE*)src;
 
     if (srcSize < ZSTDv07_frameHeaderSize_min) return ZSTDv07_frameHeaderSize_min;
+    memset(fparamsPtr, 0, sizeof(*fparamsPtr));
     if (MEM_readLE32(src) != ZSTDv07_MAGICNUMBER) {
         if ((MEM_readLE32(src) & 0xFFFFFFF0U) == ZSTDv07_MAGIC_SKIPPABLE_START) {
             if (srcSize < ZSTDv07_skippableHeaderSize) return ZSTDv07_skippableHeaderSize; /* magic number + skippable frame length */
-            memset(fparamsPtr, 0, sizeof(*fparamsPtr));
             fparamsPtr->frameContentSize = MEM_readLE32((const char *)src + 4);
             fparamsPtr->windowSize = 0; /* windowSize==0 means a frame is skippable */
             return 0;
@@ -3175,11 +3175,13 @@ size_t ZSTDv07_getFrameParams(ZSTDv07_frameParams* fparamsPtr, const void* src,
         U32 windowSize = 0;
         U32 dictID = 0;
         U64 frameContentSize = 0;
-        if ((fhdByte & 0x08) != 0) return ERROR(frameParameter_unsupported);   /* reserved bits, which must be zero */
+        if ((fhdByte & 0x08) != 0)   /* reserved bits, which must be zero */
+            return ERROR(frameParameter_unsupported);
         if (!directMode) {
             BYTE const wlByte = ip[pos++];
             U32 const windowLog = (wlByte >> 3) + ZSTDv07_WINDOWLOG_ABSOLUTEMIN;
-            if (windowLog > ZSTDv07_WINDOWLOG_MAX) return ERROR(frameParameter_unsupported);
+            if (windowLog > ZSTDv07_WINDOWLOG_MAX)
+                return ERROR(frameParameter_unsupported);
             windowSize = (1U << windowLog);
             windowSize += (windowSize >> 3) * (wlByte&7);
         }
@@ -3201,7 +3203,8 @@ size_t ZSTDv07_getFrameParams(ZSTDv07_frameParams* fparamsPtr, const void* src,
             case 3 : frameContentSize = MEM_readLE64(ip+pos); break;
         }
         if (!windowSize) windowSize = (U32)frameContentSize;
-        if (windowSize > windowSizeMax) return ERROR(frameParameter_unsupported);
+        if (windowSize > windowSizeMax)
+            return ERROR(frameParameter_unsupported);
         fparamsPtr->frameContentSize = frameContentSize;
         fparamsPtr->windowSize = windowSize;
         fparamsPtr->dictID = dictID;
@@ -3220,11 +3223,10 @@ size_t ZSTDv07_getFrameParams(ZSTDv07_frameParams* fparamsPtr, const void* src,
                    - frame header not completely provided (`srcSize` too small) */
 unsigned long long ZSTDv07_getDecompressedSize(const void* src, size_t srcSize)
 {
-    {   ZSTDv07_frameParams fparams;
-        size_t const frResult = ZSTDv07_getFrameParams(&fparams, src, srcSize);
-        if (frResult!=0) return 0;
-        return fparams.frameContentSize;
-    }
+    ZSTDv07_frameParams fparams;
+    size_t const frResult = ZSTDv07_getFrameParams(&fparams, src, srcSize);
+    if (frResult!=0) return 0;
+    return fparams.frameContentSize;
 }
 
 

From b1d9ca737a318d153eb439b4f4ed786bd0cf4384 Mon Sep 17 00:00:00 2001
From: George Lu 
Date: Mon, 13 Aug 2018 12:51:22 -0700
Subject: [PATCH 167/372] Add memoTable options

-hashing memotable
-no memotable
---
 tests/README.md    |   9 +-
 tests/paramgrill.c | 424 ++++++++++++++++++++-------------------------
 2 files changed, 197 insertions(+), 236 deletions(-)

diff --git a/tests/README.md b/tests/README.md
index 3c35ecf01..bdc9fff97 100644
--- a/tests/README.md
+++ b/tests/README.md
@@ -113,14 +113,15 @@ Full list of arguments
     dSpeed=   : Minimum decompression speed
     cMem=     : Maximum compression memory
     lvl=      : Searches for solutions which are strictly better than that compression lvl in ratio and cSpeed, 
-    stc=      : When invoked with lvl=, represents percentage slack in ratio/cSpeed allowed for a solution to be considered (Default 99%)
+    stc=      : When invoked with lvl=, represents percentage slack in ratio/cSpeed allowed for a solution to be considered (Default 100%)
               : In normal operation, represents percentage slack in choosing viable starting strategy selection in choosing the default parameters
                 (Lower value will begin with stronger strategies) (Default 90%)
-    preferSpeed= / preferRatio=
-              : Only affects lvl = invocations. Defines value placed on compression speed or ratio
-                when determining overall winner (default speed = 1, ratio = 5 for both, higher = more valued).
+    speedRatio=   (accepts decimals)
+              : determines value of gains in speed vs gains in ratio
+                when determining overall winner (default 5 (1% ratio = 5% speed)).
     tries=    : Maximum number of random restarts on a single strategy before switching (Default 3)
                 Higher values will make optimizer run longer, more chances to find better solution.
+    memLog    : Limits the log of the size of each memotable (1 per strategy). Setting memLog = 0 turns off memoization 
  -P#          : generated sample compressibility 
  -t#          : Caps runtime of operation in seconds (default : 99999 seconds (about 27 hours )) 
  -v           : Prints Benchmarking output
diff --git a/tests/paramgrill.c b/tests/paramgrill.c
index c2979cb83..4a7fc387a 100644
--- a/tests/paramgrill.c
+++ b/tests/paramgrill.c
@@ -117,10 +117,6 @@ typedef struct {
     U32 vals[NUM_PARAMS];
 } paramValues_t;
 
-/* list of parameters */
-static const varInds_t paramTable[NUM_PARAMS] = 
-        { wlog_ind, clog_ind, hlog_ind, slog_ind, slen_ind, tlen_ind, strt_ind, fadt_ind };
-
 /* maximum value of parameters */
 static const U32 mintable[NUM_PARAMS] = 
         { ZSTD_WINDOWLOG_MIN, ZSTD_CHAINLOG_MIN, ZSTD_HASHLOG_MIN, ZSTD_SEARCHLOG_MIN, ZSTD_SEARCHLENGTH_MIN, ZSTD_TARGETLENGTH_MIN, ZSTD_fast, FADT_MIN };
@@ -236,7 +232,21 @@ static U32 g_target = 0;
 static U32 g_noSeed = 0;
 static paramValues_t g_params; /* Initialized at the beginning of main w/ emptyParams() function */
 static UTIL_time_t g_time; /* to be used to compare solution finding speeds to compare to original */
-static U32 g_memoLimit = (U32)-1; //32 MB
+static U32 g_memoTableLog = PARAM_UNSET;
+
+typedef enum {
+    directMap,
+    xxhashMap,
+    noMemo
+} memoTableType_t;
+
+typedef struct {
+    memoTableType_t tableType;
+    BYTE* table;
+    size_t tableLen;
+    varInds_t varArray[NUM_PARAMS];
+    size_t varLen;
+} memoTable_t;
 
 typedef struct {
     BMK_result_t result;
@@ -264,14 +274,12 @@ static winner_ll_node* g_winners; /* linked list sorted ascending by cSize & cSp
 static BMK_result_t g_lvltarget;
 static int g_optmode = 0;
 
-static U32 g_speedMultiplier = 1;
-static U32 g_ratioMultiplier = 5;
+static double g_ratioMultiplier = 5.;
 
 /* g_mode? */
 
 /* range 0 - 99, measure of how strict  */
-#define DEFAULT_STRICTNESS 99999
-static U32 g_strictness = DEFAULT_STRICTNESS;
+static U32 g_strictness = PARAM_UNSET;
 
 void BMK_SetNbIterations(int nbLoops)
 {
@@ -281,7 +289,6 @@ void BMK_SetNbIterations(int nbLoops)
 
 /*
  * Additional Global Variables (Defined Above Use)
- * g_stratName
  * g_level_constraint
  * g_alreadyTested
  * g_maxTries
@@ -324,7 +331,7 @@ static paramValues_t cParamsToPVals(ZSTD_compressionParameters c) {
 }
 
 /* equivalent of ZSTD_adjustCParams for paramValues_t */
-static paramValues_t adjustParams(paramValues_t p, size_t maxBlockSize, size_t dictSize) {
+static paramValues_t adjustParams(paramValues_t p, const size_t maxBlockSize, const size_t dictSize) {
     paramValues_t ot = p;
     varInds_t i;
     p = cParamsToPVals(ZSTD_adjustCParams(pvalsToCParams(p), maxBlockSize, dictSize));
@@ -337,7 +344,7 @@ static paramValues_t adjustParams(paramValues_t p, size_t maxBlockSize, size_t d
 }
 
 /* accuracy in seconds only, span can be multiple years */
-static U32 BMK_timeSpan(UTIL_time_t tStart) { return (U32)(UTIL_clockSpanMicro(tStart) / 1000000ULL); }
+static U32 BMK_timeSpan(const UTIL_time_t tStart) { return (U32)(UTIL_clockSpanMicro(tStart) / 1000000ULL); }
 
 static size_t BMK_findMaxMem(U64 requiredMem)
 {
@@ -418,7 +425,7 @@ static void findClockGranularity(void) {
         return 0;                                     \
 }   }
 
-static int paramValid(paramValues_t paramTarget) {
+static int paramValid(const paramValues_t paramTarget) {
     U32 i;
     for(i = 0; i < NUM_PARAMS; i++) {
         CLAMPCHECK(paramTarget.vals[i], mintable[i], maxtable[i]);
@@ -452,7 +459,7 @@ static void BMK_translateAdvancedParams(FILE* f, const paramValues_t params) {
     fprintf(f, "\n");
 }
 
-static void BMK_displayOneResult(FILE* f, winnerInfo_t res, size_t srcSize) {
+static void BMK_displayOneResult(FILE* f, winnerInfo_t res, const size_t srcSize) {
             varInds_t v;
             res.params = cParamUnsetMin(res.params);
             fprintf(f,"    {");
@@ -498,7 +505,7 @@ static double resultDistLvl(const BMK_result_t result1, const BMK_result_t lvlRe
     if(normalizedRatioGain1 < 0 || normalizedCSpeedGain1 < 0) {
         return 0.0;
     }
-    return normalizedRatioGain1 * g_ratioMultiplier + normalizedCSpeedGain1 * g_speedMultiplier;
+    return normalizedRatioGain1 * g_ratioMultiplier + normalizedCSpeedGain1;
 }
 
 /* return true if r2 strictly better than r1 */ 
@@ -723,10 +730,10 @@ static void optimizerAdjustInput(paramValues_t* pc, const size_t maxBlockSize) {
 }
 
 /* what about low something like clog vs hlog in lvl 1?  */
-static int redundantParams(const paramValues_t paramValues, const constraint_t target, const size_t srcSize) {
+static int redundantParams(const paramValues_t paramValues, const constraint_t target, const size_t maxBlockSize) {
     return 
        (ZSTD_estimateCStreamSize_usingCParams(pvalsToCParams(paramValues)) > (size_t)target.cMem) /* Uses too much memory */
-    || ((1ULL << (paramValues.vals[wlog_ind] - 1)) >= srcSize && paramValues.vals[wlog_ind] != mintable[wlog_ind]) /* wlog too much bigger than src size */
+    || ((1ULL << (paramValues.vals[wlog_ind] - 1)) >= maxBlockSize && paramValues.vals[wlog_ind] != mintable[wlog_ind]) /* wlog too much bigger than src size */
     || (paramValues.vals[clog_ind] > (paramValues.vals[wlog_ind] + (paramValues.vals[strt_ind] > ZSTD_btlazy2))) /* chainLog larger than windowLog*/
     || (paramValues.vals[slog_ind] > paramValues.vals[clog_ind]) /* searchLog larger than chainLog */
     || (paramValues.vals[hlog_ind] > paramValues.vals[wlog_ind] + 1); /* hashLog larger than windowLog + 1 */
@@ -759,7 +766,7 @@ static void freeBuffers(const buffers_t b) {
 }
 
 /* srcBuffer will be freed by freeBuffers now */
-static int createBuffersFromMemory(buffers_t* buff, void * srcBuffer, size_t nbFiles,
+static int createBuffersFromMemory(buffers_t* buff, void * srcBuffer, const size_t nbFiles,
     const size_t* fileSizes)
 {
     size_t pos = 0, n, blockSize;
@@ -978,7 +985,6 @@ static BMK_return_t BMK_benchMemInvertible(const buffers_t buf, const contexts_t
     ZSTD_DCtx* dctx = ctx.dctx;
 
     /* warmimg up memory */
-    /* can't do this if decode only */
     for(i = 0; i < buf.nbBlocks; i++) {
         if(mode != BMK_decodeOnly) {
             RDG_genBuffer(dstPtrs[i], dstCapacities[i], 0.10, 0.50, 1);
@@ -988,7 +994,6 @@ static BMK_return_t BMK_benchMemInvertible(const buffers_t buf, const contexts_t
     }
 
     /* Bench */
-       
     {
         /* init args */
         BMK_initCCtxArgs cctxprep;
@@ -1234,21 +1239,21 @@ static void BMK_printWinner(FILE* f, const U32 cLevel, const BMK_result_t result
         snprintf(lvlstr, 15, "  Level %2u  ", cLevel);
     }
 
-    fprintf(f, "/* %s */   ", lvlstr);
-    BMK_displayOneResult(f, w, srcSize);
-
-    if(TIMED) {
+    if(TIMED) { 
         const U64 time = UTIL_clockSpanNano(g_time);
         const U64 minutes = time / (60ULL * TIMELOOP_NANOSEC);
-        fprintf(f, " - %1lu:%2lu:%05.2f", (unsigned long) minutes / 60,(unsigned long) minutes % 60, (double)(time - minutes * TIMELOOP_NANOSEC * 60ULL)/TIMELOOP_NANOSEC); 
+        fprintf(f, "%1lu:%2lu:%05.2f - ", (unsigned long) minutes / 60,(unsigned long) minutes % 60, (double)(time - minutes * TIMELOOP_NANOSEC * 60ULL)/TIMELOOP_NANOSEC); 
     }
-    fprintf(f, "\n"); 
+
+    fprintf(f, "/* %s */   ", lvlstr);
+    BMK_displayOneResult(f, w, srcSize);
 }
 
 static void BMK_printWinnerOpt(FILE* f, const U32 cLevel, const BMK_result_t result, const paramValues_t params, const constraint_t targetConstraints, const size_t srcSize)
 {
     /* global winner used for constraints */
-    static winnerInfo_t g_winner = { { (size_t)-1LL, 0, 0,  (size_t)-1LL }, { { PARAM_UNSET, PARAM_UNSET, PARAM_UNSET, PARAM_UNSET, PARAM_UNSET, PARAM_UNSET, PARAM_UNSET, PARAM_UNSET } } };
+                                    /* cSize, cSpeed, dSpeed, cMem */
+    static winnerInfo_t g_winner = { { (size_t)-1LL, 0, 0, (size_t)-1LL }, { { PARAM_UNSET, PARAM_UNSET, PARAM_UNSET, PARAM_UNSET, PARAM_UNSET, PARAM_UNSET, PARAM_UNSET, PARAM_UNSET } } };
     if(DEBUG || compareResultLT(g_winner.result, result, targetConstraints, srcSize)) {
         if(DEBUG && compareResultLT(g_winner.result, result, targetConstraints, srcSize)) {
             DISPLAY("New Winner: \n");
@@ -1445,9 +1450,7 @@ static int BMK_seed(oldWinnerInfo_t* winners, const ZSTD_compressionParameters p
 }
 
 /* nullified useless params, to ensure count stats */
-/* no point in windowLog < chainLog (no point 2x chainLog for bt) */
-/* now with built in bounds-checking */
-/* no longer does anything with sanitizeVarArray + clampcheck */
+/* cleans up params for memoizing / display */
 static paramValues_t sanitizeParams(paramValues_t params)
 {
     if (params.vals[strt_ind] == ZSTD_fast)
@@ -1463,8 +1466,8 @@ static paramValues_t sanitizeParams(paramValues_t params)
 /* return: new length */
 /* keep old array, will need if iter over strategy. */
 /* prunes useless params */
-static int sanitizeVarArray(varInds_t* varNew, const int varLength, const varInds_t* varArray, const ZSTD_strategy strat) {
-    int i, j = 0;
+static size_t sanitizeVarArray(varInds_t* varNew, const size_t varLength, const varInds_t* varArray, const ZSTD_strategy strat) {
+    size_t i, j = 0;
     for(i = 0; i < varLength; i++) {
         if( !((varArray[i] == clog_ind && strat == ZSTD_fast)
             || (varArray[i] == slog_ind && strat == ZSTD_fast)
@@ -1481,9 +1484,9 @@ static int sanitizeVarArray(varInds_t* varNew, const int varLength, const varInd
 /* res should be NUM_PARAMS size */
 /* constructs varArray from paramValues_t style parameter */
 /* pass in using dict. */
-static int variableParams(const paramValues_t paramConstraints, varInds_t* res, const int usingDictionary) {
+static size_t variableParams(const paramValues_t paramConstraints, varInds_t* res, const int usingDictionary) {
     varInds_t i;
-    int j = 0;
+    size_t j = 0;
     for(i = 0; i < NUM_PARAMS; i++) {
         if(paramConstraints.vals[i] == PARAM_UNSET) {
             if(i == fadt_ind && !usingDictionary) continue; /* don't use fadt if no dictionary */
@@ -1501,7 +1504,7 @@ static void paramVaryOnce(const varInds_t paramIndex, const int amt, paramValues
 }
 
 /* varies ptr by nbChanges respecting varyParams*/
-static void paramVariation(paramValues_t* ptr, const varInds_t* varyParams, const int varyLen, const U32 nbChanges)
+static void paramVariation(paramValues_t* ptr, memoTable_t* mtAll, const U32 nbChanges)
 {
     paramValues_t p;
     U32 validated = 0;
@@ -1509,8 +1512,8 @@ static void paramVariation(paramValues_t* ptr, const varInds_t* varyParams, cons
         U32 i;
         p = *ptr;
         for (i = 0 ; i < nbChanges ; i++) {
-            const U32 changeID = FUZ_rand(&g_rand) % (varyLen << 1);
-            paramVaryOnce(varyParams[changeID >> 1], ((changeID & 1) << 1) - 1, &p);
+            const U32 changeID = (U32)FUZ_rand(&g_rand) % (mtAll[p.vals[strt_ind]].varLen << 1);
+            paramVaryOnce(mtAll[p.vals[strt_ind]].varArray[changeID >> 1], ((changeID & 1) << 1) - 1, &p);
         }
         validated = paramValid(p);
     }
@@ -1518,9 +1521,9 @@ static void paramVariation(paramValues_t* ptr, const varInds_t* varyParams, cons
 }
 
 /* length of memo table given free variables */
-static size_t memoTableLen(const varInds_t* varyParams, const int varyLen) {
+static size_t memoTableLen(const varInds_t* varyParams, const size_t varyLen) {
     size_t arrayLen = 1;
-    int i;
+    size_t i;
     for(i = 0; i < varyLen; i++) {
         if(varyParams[i] == strt_ind) continue; /* strategy separated by table */
         arrayLen *= rangetable[varyParams[i]];
@@ -1529,8 +1532,8 @@ static size_t memoTableLen(const varInds_t* varyParams, const int varyLen) {
 }
 
 /* returns unique index in memotable of compression parameters */
-static unsigned memoTableInd(const paramValues_t* ptr, const varInds_t* varyParams, const int varyLen) {
-    int i;
+static unsigned memoTableIndDirect(const paramValues_t* ptr, const varInds_t* varyParams, const size_t varyLen) {
+    size_t i;
     unsigned ind = 0;
     for(i = 0; i < varyLen; i++) {
         varInds_t v = varyParams[i];
@@ -1540,84 +1543,82 @@ static unsigned memoTableInd(const paramValues_t* ptr, const varInds_t* varyPara
     return ind;
 }
 
-/* inverse of above function (from index to parameters) */
-static void memoTableIndInv(paramValues_t* ptr, const varInds_t* varyParams, const int varyLen, size_t ind) {
-    int i;
-    for(i = varyLen - 1; i >= 0; i--) {
-        /* This is cleaner/easier to generalize but slower
-        varInds_t v = varyParams[i];
-        if(v == strt_ind) continue;
-        ptr->vals[v] = rangeMap(v, ind % rangetable[v]);
-        ind /= rangetable[v]; */
-
-        switch(varyParams[i]) {
-            case wlog_ind: ptr->vals[wlog_ind] = ind % rangetable[wlog_ind] + mintable[wlog_ind]; ind /= rangetable[wlog_ind]; break;
-            case clog_ind: ptr->vals[clog_ind] = ind % rangetable[clog_ind] + mintable[clog_ind]; ind /= rangetable[clog_ind]; break;
-            case hlog_ind: ptr->vals[hlog_ind] = ind % rangetable[hlog_ind] + mintable[hlog_ind]; ind /= rangetable[hlog_ind]; break;
-            case slog_ind: ptr->vals[slog_ind] = ind % rangetable[slog_ind] + mintable[slog_ind]; ind /= rangetable[slog_ind]; break;
-            case slen_ind: ptr->vals[slen_ind] = ind % rangetable[slen_ind] + mintable[slen_ind]; ind /= rangetable[slen_ind]; break;
-            case tlen_ind: ptr->vals[tlen_ind] = tlen_table[(ind % rangetable[tlen_ind])];        ind /= rangetable[tlen_ind]; break;
-            case fadt_ind: ptr->vals[fadt_ind] = ind % rangetable[fadt_ind] - 1;                  ind /= rangetable[fadt_ind]; break;
-            case strt_ind:
-            case NUM_PARAMS: break;
-        }
+static size_t memoTableGet(const memoTable_t* memoTableArray, const paramValues_t p) {
+    const memoTable_t mt = memoTableArray[p.vals[strt_ind]];
+    switch(mt.tableType) {
+        case directMap:
+            return mt.table[memoTableIndDirect(&p, mt.varArray, mt.varLen)];
+        case xxhashMap:
+            return mt.table[(XXH64(&p.vals, sizeof(U32) * NUM_PARAMS, 0) >> 3) % mt.tableLen];
+        case noMemo:
+            return 0;
     }
+    return 0; /* should never happen, stop compiler warnings */
 }
 
-/* Initialize memoization table, which tracks and prevents repeated benchmarking
- * of the same set of parameters. In addition, it is also used to immediately mark 
- * redundant / obviously non-optimal parameter configurations (e.g. wlog - 1 larger)
- * than srcSize, clog > wlog, ... 
- */
-static void initMemoTable(U8* memoTable, paramValues_t paramConstraints, const constraint_t target, const varInds_t* varyParams, const int varyLen, const size_t srcSize) {
-    size_t i;
-    size_t arrayLen = memoTableLen(varyParams, varyLen);
-    int j = 0;
-    assert(memoTable != NULL);
-    memset(memoTable, 0, arrayLen);
-    paramConstraints = cParamUnsetMin(paramConstraints);
-
-    for(i = 0; i < arrayLen; i++) {
-        memoTableIndInv(¶mConstraints, varyParams, varyLen, i);
-        if(redundantParams(paramConstraints, target, srcSize)) {
-            memoTable[i] = 255; j++;
-        }
+static void memoTableSet(const memoTable_t* memoTableArray, const paramValues_t p, const BYTE value) {
+    const memoTable_t mt = memoTableArray[p.vals[strt_ind]];
+    switch(mt.tableType) {
+        case directMap:
+            mt.table[memoTableIndDirect(&p, mt.varArray, mt.varLen)] = value; break;
+        case xxhashMap:
+            mt.table[(XXH64(&p.vals, sizeof(U32) * NUM_PARAMS, 0) >> 3) % mt.tableLen] = value; break;
+        case noMemo:
+            break;
     }
-
-    DEBUGOUTPUT("%d / %d Invalid\n", j, (int)arrayLen);
 }
 
 /* frees all allocated memotables */
-static void freeMemoTableArray(U8** mtAll) {
+static void freeMemoTableArray(memoTable_t* const mtAll) {
     int i;
     if(mtAll == NULL) { return; }
     for(i = 1; i <= (int)ZSTD_btultra; i++) {
-        free(mtAll[i]);
+        free(mtAll[i].table);
     }
     free(mtAll);
 }
 
 /* inits memotables for all (including mallocs), all strategies */
 /* takes unsanitized varyParams */
-static U8** createMemoTableArray(paramValues_t paramConstraints, const constraint_t target, const varInds_t* varyParams, const int varyLen, const size_t srcSize) {
-    varInds_t varNew[NUM_PARAMS];
-    U8** mtAll = (U8**)calloc(sizeof(U8*),(ZSTD_btultra + 1));
+static memoTable_t* createMemoTableArray(const varInds_t* const varyParams, const size_t varyLen) {
+    memoTable_t* mtAll = (memoTable_t*)calloc(sizeof(memoTable_t),(ZSTD_btultra + 1));
     int i;
+    
     if(mtAll == NULL) {
         return NULL;
     }
 
     for(i = 1; i <= (int)ZSTD_btultra; i++) {
-        const int varLenNew = sanitizeVarArray(varNew, varyLen, varyParams, i);
-        size_t mtl = memoTableLen(varNew, varLenNew);
-        if(mtl > g_memoLimit) { mtAll[i] = NULL; continue; }
-        mtAll[i] = malloc(sizeof(U8) * mtl);
-        if(mtAll[i] == NULL) {
+        mtAll[i].varLen = sanitizeVarArray(mtAll[i].varArray, varyLen, varyParams, i);
+    }
+
+    /* no memoization */
+    if(g_memoTableLog == 0) {
+        for(i = 1; i <= (int)ZSTD_btultra; i++) {
+            mtAll[i].tableType = noMemo;
+            mtAll[i].table = NULL;
+            mtAll[i].tableLen = 0;
+        }
+        return mtAll;
+    }
+
+    /* hash table if normal table is too big */
+    for(i = 1; i <= (int)ZSTD_btultra; i++) {
+        size_t mtl = memoTableLen(mtAll[i].varArray, mtAll[i].varLen);
+        mtAll[i].tableType = directMap;
+
+        if(g_memoTableLog != PARAM_UNSET && mtl > (1ULL << g_memoTableLog)) { /* use hash table */ /* provide some option to only use hash tables? */
+            mtAll[i].tableType = xxhashMap;
+            mtl = (1ULL << g_memoTableLog);
+        }
+
+        mtAll[i].table = (BYTE*)calloc(sizeof(BYTE), mtl);
+        mtAll[i].tableLen = mtl;
+
+        if(mtAll[i].table == NULL) {
             freeMemoTableArray(mtAll);
             return NULL;
         }
-        paramConstraints.vals[strt_ind] = i;
-        initMemoTable(mtAll[i], paramConstraints, target, varNew, varLenNew, srcSize);
     }
     
     return mtAll;
@@ -1638,10 +1639,6 @@ static paramValues_t overwriteParams(paramValues_t base, const paramValues_t mas
 #define PARAMTABLEMASK (PARAMTABLESIZE-1)
 static BYTE g_alreadyTested[PARAMTABLESIZE] = {0};   /* init to zero */
 
-/*
-#define NB_TESTS_PLAYED(p) \
-    g_alreadyTested[(XXH64(((void*)&sanitizeParams(p), sizeof(p), 0) >> 3) & PARAMTABLEMASK] */
-
 static BYTE* NB_TESTS_PLAYED(ZSTD_compressionParameters p) {
     ZSTD_compressionParameters p2 = pvalsToCParams(sanitizeParams(cParamsToPVals(p)));
     return &g_alreadyTested[(XXH64((void*)&p2, sizeof(p2), 0) >> 3) & PARAMTABLEMASK];
@@ -1651,10 +1648,8 @@ static void playAround(FILE* f, oldWinnerInfo_t* winners,
                        ZSTD_compressionParameters params,
                        const buffers_t buf, const contexts_t ctx)
 {
-    int nbVariations = 0;
+    int nbVariations = 0, i;
     UTIL_time_t const clockStart = UTIL_getTime();
-    const U32 unconstrained[NUM_PARAMS] = { 0, 1, 2, 3, 4, 5, 6 }; /* no fadt */
-
 
     while (UTIL_clockSpanMicro(clockStart) < g_maxVariationTime) {
         paramValues_t p = cParamsToPVals(params);
@@ -1662,7 +1657,9 @@ static void playAround(FILE* f, oldWinnerInfo_t* winners,
         BYTE* b;
 
         if (nbVariations++ > g_maxNbVariations) break;
-        paramVariation(&p, unconstrained, NUM_PARAMS, 4);
+
+        do { for(i = 0; i < 4; i++) { paramVaryOnce(FUZ_rand(&g_rand) % (strt_ind + 1), ((FUZ_rand(&g_rand) & 1) << 1) - 1, &p); } } 
+        while(!paramValid(p));
 
         p2 = pvalsToCParams(p);
 
@@ -1683,28 +1680,31 @@ static void playAround(FILE* f, oldWinnerInfo_t* winners,
 }
 
 /* Sets pc to random unmeasured set of parameters */
-/* Doesn't do strategy! */
-static void randomConstrainedParams(paramValues_t* pc, const varInds_t* varArray, const int varLen, const U8* memoTable)
+/* specifiy strategy */
+static void randomConstrainedParams(paramValues_t* pc, const memoTable_t* memoTableArray, const ZSTD_strategy st)
 {
     size_t j;
-    for(j = 0; j < MIN(g_memoLimit, memoTableLen(varArray, varLen)); j++) {
+    const memoTable_t mt = memoTableArray[st];
+    pc->vals[strt_ind] = st;
+    for(j = 0; j < MIN(1ULL << g_memoTableLog, memoTableLen(mt.varArray, mt.varLen)); j++) {
         int i;
         for(i = 0; i < NUM_PARAMS; i++) {
-            varInds_t v = varArray[i];
-            if(v == strt_ind) continue; //don't do strategy (dependent on MT)
+            varInds_t v = mt.varArray[i];
+            if(v == strt_ind) continue; //skip, already specified
             pc->vals[v] = rangeMap(v, FUZ_rand(&g_rand) % rangetable[v]);
         }
 
-        if(memoTable == NULL || memoTable[memoTableInd(pc, varArray, varLen)]) { break; }
+        if(!(memoTableGet(memoTableArray, *pc))) break; //only pick unpicked params. 
     }
 }
 
 /* Completely random parameter selection */
 static ZSTD_compressionParameters randomParams(void)
 {
-    paramValues_t p;
-    p.vals[strt_ind] = rangeMap(strt_ind, rangeMap(strt_ind, FUZ_rand(&g_rand) % rangetable[strt_ind]));
-    randomConstrainedParams(&p, paramTable, NUM_PARAMS, NULL);
+    paramValues_t p; varInds_t v;
+    for(v = 0; v < NUM_PARAMS; v++) {
+        p.vals[v] = rangeMap(v, FUZ_rand(&g_rand) % rangetable[v]);
+    }
     return pvalsToCParams(p);
 }
 
@@ -1866,68 +1866,62 @@ int benchFiles(const char** fileNamesTable, int nbFiles, const char* dictFileNam
     return ret;
 }
 
+#define CBENCHMARK(conditional, resultvar, tmpret, mode, loopmode, sec) {                                       \
+    if(conditional) {                                                                                           \
+        BMK_return_t tmpret = BMK_benchMemInvertible(buf, ctx, BASE_CLEVEL, &cParams, mode, loopmode, sec);     \
+        if(tmpret.error) { DEBUGOUTPUT("Benchmarking failed\n"); return ERROR_RESULT; }                         \
+        if(mode != BMK_decodeOnly)  {                                                                           \
+            resultvar.cSpeed = tmpret.result.cSpeed;                                                            \
+            resultvar.cSize = tmpret.result.cSize;                                                              \
+            resultvar.cMem = tmpret.result.cMem;                                                                \
+        }                                                                                                       \
+        if(mode != BMK_compressOnly) { resultvar.dSpeed = tmpret.result.dSpeed; }                               \
+    }                                                                                                           \
+}
+
 /* Benchmarking which stops when we are sufficiently sure the solution is infeasible / worse than the winner */
 #define VARIANCE 1.2 
-#define HIGH_VARIANCE 100.0
 static int allBench(BMK_result_t* resultPtr,
                 const buffers_t buf, const contexts_t ctx,
                 const paramValues_t cParams,
                 const constraint_t target,
                 BMK_result_t* winnerResult, int feas) {
-    BMK_return_t benchres;
-    BMK_result_t resultMax;
+    BMK_result_t resultMax, benchres;
     U64 loopDurationC = 0, loopDurationD = 0;
     double uncertaintyConstantC = 3., uncertaintyConstantD = 3.;
     double winnerRS;
-
     /* initial benchmarking, gives exact ratio and memory, warms up future runs */
-    benchres = BMK_benchMemInvertible(buf, ctx, BASE_CLEVEL, &cParams, BMK_both, BMK_iterMode, 1);
+    CBENCHMARK(1, benchres, tmp, BMK_both, BMK_iterMode, 1);
 
     winnerRS = resultScore(*winnerResult, buf.srcSize, target);
     DEBUGOUTPUT("WinnerScore: %f\n ", winnerRS);
 
-    if(benchres.error) {
-        DEBUGOUTPUT("Benchmarking failed\n");
-        return ERROR_RESULT;
-    } 
-    *resultPtr = benchres.result;
+    *resultPtr = benchres;
 
     /* calculate uncertainty in compression / decompression runs */
-    if(benchres.result.cSpeed) {
-        loopDurationC = ((buf.srcSize * TIMELOOP_NANOSEC) / benchres.result.cSpeed); 
+    if(benchres.cSpeed) {
+        loopDurationC = ((buf.srcSize * TIMELOOP_NANOSEC) / benchres.cSpeed); 
         uncertaintyConstantC = ((loopDurationC + (double)(2 * g_clockGranularity))/loopDurationC); 
     }
 
-    if(benchres.result.dSpeed) {
-        loopDurationD = ((buf.srcSize * TIMELOOP_NANOSEC) / benchres.result.dSpeed); 
+    if(benchres.dSpeed) {
+        loopDurationD = ((buf.srcSize * TIMELOOP_NANOSEC) / benchres.dSpeed); 
         uncertaintyConstantD = ((loopDurationD + (double)(2 * g_clockGranularity))/loopDurationD);  
     }
 
     /* anything with worse ratio in feas is definitely worse, discard */
-    if(feas && benchres.result.cSize < winnerResult->cSize && !g_optmode) {
+    if(feas && benchres.cSize < winnerResult->cSize && !g_optmode) {
         return WORSE_RESULT;
     }
 
     /* second run, if first run is too short, gives approximate cSpeed + dSpeed */
-    if(loopDurationC < TIMELOOP_NANOSEC / 10) {
-        BMK_return_t benchres2 = BMK_benchMemInvertible(buf, ctx, BASE_CLEVEL, &cParams, BMK_compressOnly, BMK_iterMode, 1);
-        if(benchres2.error) {
-            return ERROR_RESULT;
-        }
-        benchres = benchres2;
-    }
-    if(loopDurationD < TIMELOOP_NANOSEC / 10) {
-        BMK_return_t benchres2 = BMK_benchMemInvertible(buf, ctx, BASE_CLEVEL, &cParams, BMK_decodeOnly, BMK_iterMode, 1);
-        if(benchres2.error) {
-            return ERROR_RESULT;
-        }
-        benchres.result.dSpeed = benchres2.result.dSpeed;
-    }
+    CBENCHMARK(loopDurationC < TIMELOOP_NANOSEC / 10, benchres, tmp, BMK_compressOnly, BMK_iterMode, 1);
+    CBENCHMARK(loopDurationD < TIMELOOP_NANOSEC / 10, benchres, tmp, BMK_decodeOnly,   BMK_iterMode, 1);
 
-    *resultPtr = benchres.result;
+    *resultPtr = benchres;
 
-    /* optimistic assumption of benchres.result */
-    resultMax = benchres.result;
+    /* optimistic assumption of benchres */
+    resultMax = benchres;
     resultMax.cSpeed *= uncertaintyConstantC * VARIANCE;
     resultMax.dSpeed *= uncertaintyConstantD * VARIANCE;
 
@@ -1938,29 +1932,15 @@ static int allBench(BMK_result_t* resultPtr,
         return WORSE_RESULT;
     }
 
-    /* Final full run if estimates are unclear */
-    if(loopDurationC < TIMELOOP_NANOSEC) {
-        BMK_return_t benchres2 = BMK_benchMemInvertible(buf, ctx, BASE_CLEVEL, &cParams, BMK_compressOnly, BMK_timeMode, 1);
-        if(benchres2.error) {
-            return ERROR_RESULT;
-        }
-        benchres.result.cSpeed = benchres2.result.cSpeed;
-    } 
+    CBENCHMARK(loopDurationC < TIMELOOP_NANOSEC, benchres, tmp, BMK_compressOnly, BMK_timeMode, 1);
+    CBENCHMARK(loopDurationD < TIMELOOP_NANOSEC, benchres, tmp, BMK_decodeOnly,   BMK_timeMode, 1);
 
-    if(loopDurationD < TIMELOOP_NANOSEC) {
-        BMK_return_t benchres2 = BMK_benchMemInvertible(buf, ctx, BASE_CLEVEL, &cParams, BMK_decodeOnly, BMK_timeMode, 1);
-        if(benchres2.error) {
-            return ERROR_RESULT;
-        }
-        benchres.result.dSpeed = benchres2.result.dSpeed;
-    }
-
-    *resultPtr = benchres.result;
+    *resultPtr = benchres;
 
     /* compare by resultScore when in infeas */
     /* compare by compareResultLT when in feas */
-    if((!feas && (resultScore(benchres.result, buf.srcSize, target) > resultScore(*winnerResult, buf.srcSize, target))) || 
-       (feas && (compareResultLT(*winnerResult, benchres.result, target, buf.srcSize))) )  { 
+    if((!feas && (resultScore(benchres, buf.srcSize, target) > resultScore(*winnerResult, buf.srcSize, target))) || 
+       (feas && (compareResultLT(*winnerResult, benchres, target, buf.srcSize))) )  { 
         return BETTER_RESULT; 
     } else { 
         return WORSE_RESULT; 
@@ -1968,19 +1948,17 @@ static int allBench(BMK_result_t* resultPtr,
 }
 
 #define INFEASIBLE_THRESHOLD 200
-
 /* Memoized benchmarking, won't benchmark anything which has already been benchmarked before. */
 static int benchMemo(BMK_result_t* resultPtr,
                 const buffers_t buf, const contexts_t ctx, 
                 const paramValues_t cParams,
                 const constraint_t target,
-                BMK_result_t* winnerResult, U8* const memoTable,
-                const varInds_t* varyParams, const int varyLen, const int feas) {
+                BMK_result_t* winnerResult, memoTable_t* const memoTableArray,
+                const int feas) {
     static int bmcount = 0;
-    size_t memind = memoTableInd(&cParams, varyParams, varyLen);
     int res;
 
-    if((memoTable == NULL && redundantParams(cParams, target, buf.maxBlockSize)) || ((memoTable != NULL) && memoTable[memind] >= INFEASIBLE_THRESHOLD)) { return WORSE_RESULT; } 
+    if(memoTableGet(memoTableArray, cParams) >= INFEASIBLE_THRESHOLD || redundantParams(cParams, target, buf.maxBlockSize)) { return WORSE_RESULT; } 
 
     res = allBench(resultPtr, buf, ctx, cParams, target, winnerResult, feas);
 
@@ -1990,8 +1968,8 @@ static int benchMemo(BMK_result_t* resultPtr,
     }
     BMK_printWinnerOpt(stdout, CUSTOM_LEVEL, *resultPtr, cParams, target, buf.srcSize);
 
-    if(memoTable != NULL && (res == BETTER_RESULT || feas)) {
-        memoTable[memind] = 255; 
+    if(res == BETTER_RESULT || feas) {
+        memoTableSet(memoTableArray, cParams, 255); /* what happens if collisions are frequent */
     }
     return res;
 }
@@ -2013,8 +1991,7 @@ static int benchMemo(BMK_result_t* resultPtr,
  * all generation after random should be sanitized. (maybe sanitize random)
  */
 static winnerInfo_t climbOnce(const constraint_t target, 
-                const varInds_t* varArray, const int varLen, ZSTD_strategy strat,
-                U8** memoTableArray, 
+                memoTable_t* mtAll, 
                 const buffers_t buf, const contexts_t ctx,
                 const paramValues_t init) {
     /* 
@@ -2026,8 +2003,6 @@ static winnerInfo_t climbOnce(const constraint_t target,
     winnerInfo_t candidateInfo, winnerInfo;
     int better = 1;
     int feas = 0;
-    varInds_t varNew[NUM_PARAMS];
-    int varLenNew = sanitizeVarArray(varNew, varLen, varArray, strat);
 
     winnerInfo = initWinnerInfo(init);
     candidateInfo = winnerInfo;
@@ -2037,7 +2012,9 @@ static winnerInfo_t climbOnce(const constraint_t target,
         DEBUGOUTPUT("Climb Part 1\n");
         while(better) {
 
-            int i, dist, offset;
+            int offset;
+            size_t i, dist;
+            const size_t varLen = mtAll[cparam.vals[strt_ind]].varLen;
             better = 0;
             DEBUGOUTPUT("Start\n");
             cparam = winnerInfo.params;
@@ -2048,17 +2025,13 @@ static winnerInfo_t climbOnce(const constraint_t target,
                 for(offset = -1; offset <= 1; offset += 2) {
                     CHECKTIME(winnerInfo);
                     candidateInfo.params = cparam;
-                    paramVaryOnce(varArray[i], offset, &candidateInfo.params); 
+                    paramVaryOnce(mtAll[cparam.vals[strt_ind]].varArray[i], offset, &candidateInfo.params); 
                     
                     if(paramValid(candidateInfo.params)) {
                         int res;
-                        if(strat != candidateInfo.params.vals[strt_ind]) { /* maybe only try strategy switching after exhausting non-switching solutions? */
-                            strat = candidateInfo.params.vals[strt_ind];
-                            varLenNew = sanitizeVarArray(varNew, varLen, varArray, strat);
-                        }
                         res = benchMemo(&candidateInfo.result, buf, ctx,
-                            sanitizeParams(candidateInfo.params), target, &winnerInfo.result, memoTableArray[strat],
-                            varNew, varLenNew, feas);
+                            sanitizeParams(candidateInfo.params), target, &winnerInfo.result, mtAll, feas);
+                        DEBUGOUTPUT("Res: %d\n", res);
                         if(res == BETTER_RESULT) { /* synonymous with better when called w/ infeasibleBM */
                             winnerInfo = candidateInfo;
                             BMK_printWinnerOpt(stdout, CUSTOM_LEVEL, winnerInfo.result, sanitizeParams(winnerInfo.params), target, buf.srcSize);
@@ -2081,16 +2054,11 @@ static winnerInfo_t climbOnce(const constraint_t target,
                     CHECKTIME(winnerInfo);
                     candidateInfo.params = cparam;
                     /* param error checking already done here */
-                    paramVariation(&candidateInfo.params, varArray, varLen, dist);
-
-                    if(strat != candidateInfo.params.vals[strt_ind]) {
-                        strat = candidateInfo.params.vals[strt_ind];
-                        varLenNew = sanitizeVarArray(varNew, varLen, varArray, strat);
-                    }
+                    paramVariation(&candidateInfo.params, mtAll, (U32)dist);
 
                     res = benchMemo(&candidateInfo.result, buf, ctx, 
-                        sanitizeParams(candidateInfo.params), target, &winnerInfo.result, memoTableArray[strat],
-                        varNew, varLenNew, feas);
+                        sanitizeParams(candidateInfo.params), target, &winnerInfo.result, mtAll, feas);
+                    DEBUGOUTPUT("Res: %d\n", res);
                     if(res == BETTER_RESULT) { /* synonymous with better in this case*/
                         winnerInfo = candidateInfo;
                         BMK_printWinnerOpt(stdout, CUSTOM_LEVEL, winnerInfo.result, sanitizeParams(winnerInfo.params), target, buf.srcSize);
@@ -2134,11 +2102,8 @@ static winnerInfo_t optimizeFixedStrategy(
     const buffers_t buf, const contexts_t ctx, 
     const constraint_t target, paramValues_t paramTarget,
     const ZSTD_strategy strat, 
-    const varInds_t* varArray, const int varLen,
-    U8** memoTableArray, const int tries) {
+    memoTable_t* memoTableArray, const int tries) {
     int i = 0;
-    varInds_t varNew[NUM_PARAMS];
-    int varLenNew = sanitizeVarArray(varNew, varLen, varArray, strat);
 
     paramValues_t init;
     winnerInfo_t winnerInfo, candidateInfo; 
@@ -2150,14 +2115,15 @@ static winnerInfo_t optimizeFixedStrategy(
 
     init = paramTarget;
 
-    while(i < tries) {
+    for(i = 0; i < tries; i++) {
         DEBUGOUTPUT("Restart\n"); 
-        randomConstrainedParams(&init, varNew, varLenNew, memoTableArray[strat]);
-        candidateInfo = climbOnce(target, varArray, varLen, strat, memoTableArray, buf, ctx, init);
+        do { randomConstrainedParams(&init, memoTableArray, strat); } while(redundantParams(init, target, buf.maxBlockSize)); //only non-redundant params  
+        candidateInfo = climbOnce(target, memoTableArray, buf, ctx, init);
         if(compareResultLT(winnerInfo.result, candidateInfo.result, target, buf.srcSize)) {
             winnerInfo = candidateInfo;
             BMK_printWinnerOpt(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, target, buf.srcSize);
             i = 0;
+            continue;
         }
         CHECKTIME(winnerInfo);
         i++;
@@ -2217,9 +2183,9 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_
 {
     varInds_t varArray [NUM_PARAMS];
     int ret = 0;
-    const int varLen = variableParams(paramTarget, varArray, dictFileName != NULL);
+    const size_t varLen = variableParams(paramTarget, varArray, dictFileName != NULL);
     winnerInfo_t winner = initWinnerInfo(emptyParams());
-    U8** allMT = NULL;
+    memoTable_t* allMT = NULL;
     paramValues_t paramBase;
     contexts_t ctx;
     buffers_t buf;
@@ -2246,38 +2212,19 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_
     optimizerAdjustInput(¶mTarget, buf.maxBlockSize);
     paramBase = cParamUnsetMin(paramTarget);
 
-    /* if strategy is fixed, only init that part of memotable */
-    if(paramTarget.vals[strt_ind] != PARAM_UNSET) {
-        varInds_t varNew[NUM_PARAMS];
-        int varLenNew = sanitizeVarArray(varNew, varLen, varArray, paramTarget.vals[strt_ind]);
-        allMT = (U8**)calloc(sizeof(U8*), (ZSTD_btultra + 1));
-        if(allMT == NULL) {
-            ret = 57;
-            goto _cleanUp;
-        }
+    // TODO: if strategy is fixed, only init that row
+    allMT = createMemoTableArray(varArray, varLen);
 
-        allMT[paramTarget.vals[strt_ind]] = malloc(sizeof(U8) * memoTableLen(varNew, varLenNew));
-        
-        if(allMT[paramTarget.vals[strt_ind]] == NULL) {
-            ret = 58;
-            goto _cleanUp;
-        }
-        
-        initMemoTable(allMT[paramTarget.vals[strt_ind]], paramTarget, target, varNew, varLenNew, buf.maxBlockSize);
-    } else {
-         allMT = createMemoTableArray(paramTarget, target, varArray, varLen, buf.maxBlockSize);
-    }
-   
     if(!allMT) {
         DISPLAY("MemoTable Init Error\n");
         ret = 2;
         goto _cleanUp;
     }
 
-    /* default strictness = Maximum for */
-    if(g_strictness == DEFAULT_STRICTNESS) {
+    /* default strictnesses */
+    if(g_strictness == PARAM_UNSET) {
         if(g_optmode) {
-            g_strictness = 99;
+            g_strictness = 100;
         } else {
             g_strictness = 90;
         }
@@ -2371,7 +2318,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_
         }
 
         DEBUGOUTPUT("Real Opt\n");
-        /* start 'real' tests */
+        /* start 'real' optimization */
         {   
             int bestStrategy = (int)winner.params.vals[strt_ind];
             if(paramTarget.vals[strt_ind] == PARAM_UNSET) {
@@ -2380,8 +2327,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_
 
                 { 
                     /* one iterations of hill climbing with the level-defined parameters. */
-                    winnerInfo_t w1 = climbOnce(target, varArray, varLen, st, allMT, 
-                        buf, ctx, winner.params);
+                    winnerInfo_t w1 = climbOnce(target, allMT, buf, ctx, winner.params);
                     if(compareResultLT(winner.result, w1.result, target, buf.srcSize)) {
                         winner = w1;
                     }
@@ -2392,8 +2338,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_
                     winnerInfo_t wc;
                     DEBUGOUTPUT("StrategySwitch: %s\n", g_stratName[st]);
                     
-                    wc = optimizeFixedStrategy(buf, ctx, target, paramBase, 
-                        st, varArray, varLen, allMT, tries);
+                    wc = optimizeFixedStrategy(buf, ctx, target, paramBase, st, allMT, tries);
 
                     if(compareResultLT(winner.result, wc.result, target, buf.srcSize)) {
                         winner = wc;
@@ -2406,8 +2351,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_
                     CHECKTIMEGT(ret, 0, _cleanUp);
                 }
             } else {
-                winner = optimizeFixedStrategy(buf, ctx, target, paramBase, paramTarget.vals[strt_ind], 
-                    varArray, varLen, allMT, g_maxTries);
+                winner = optimizeFixedStrategy(buf, ctx, target, paramBase, paramTarget.vals[strt_ind], allMT, g_maxTries);
             }
 
         }
@@ -2467,6 +2411,22 @@ static unsigned readU32FromChar(const char** stringPtr)
     return result * sign;
 }
 
+static double readDoubleFromChar(const char** stringPtr)
+{
+    double result = 0, divide = 10;
+    while ((**stringPtr >='0') && (**stringPtr <='9')) {
+        result *= 10, result += **stringPtr - '0', (*stringPtr)++ ;
+    }
+    if(**stringPtr!='.') {
+        return result;
+    }
+    (*stringPtr)++;
+    while ((**stringPtr >='0') && (**stringPtr <='9')) {
+        result += (double)(**stringPtr - '0') / divide, divide *= 10, (*stringPtr)++ ; 
+    }
+    return result;
+}
+
 static int usage(const char* exename)
 {
     DISPLAY( "Usage :\n");
@@ -2558,11 +2518,10 @@ int main(int argc, const char** argv)
                 PARSE_SUB_ARGS("decompressionSpeed=", "dSpeed=", target.dSpeed); 
                 PARSE_SUB_ARGS("compressionMemory=" , "cMem=", target.cMem);
                 PARSE_SUB_ARGS("strict=", "stc=", g_strictness);
-                PARSE_SUB_ARGS("preferSpeed=", "prfSpd=", g_speedMultiplier);
-                PARSE_SUB_ARGS("preferRatio=", "prfRto=", g_ratioMultiplier);
                 PARSE_SUB_ARGS("maxTries=", "tries=", g_maxTries);
-                PARSE_SUB_ARGS("memoLimit=", "memo=", g_memoLimit);
+                PARSE_SUB_ARGS("memoLimitLog=", "memLog=", g_memoTableLog);
                 if (longCommandWArg(&argument, "level=") || longCommandWArg(&argument, "lvl=")) { cLevelOpt = readU32FromChar(&argument); g_optmode = 1; if (argument[0]==',') { argument++; continue; } else break; }
+                if (longCommandWArg(&argument, "speedForRatio=") || longCommandWArg(&argument, "speedRatio=")) { g_ratioMultiplier = readDoubleFromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; }
 
                 DISPLAY("invalid optimization parameter \n");
                 return 1;
@@ -2690,6 +2649,7 @@ int main(int argc, const char** argv)
                     break;
 
                 case 's':
+                    argument++;
                     seperateFiles = 1;
                     break;
 

From 3dcfe5cc2c19b5a4fd864c5d1c40aec4a9e5f1dc Mon Sep 17 00:00:00 2001
From: George Lu 
Date: Tue, 14 Aug 2018 14:24:05 -0700
Subject: [PATCH 168/372] begin display changes

---
 tests/paramgrill.c | 14 +++++++++++---
 1 file changed, 11 insertions(+), 3 deletions(-)

diff --git a/tests/paramgrill.c b/tests/paramgrill.c
index 4a7fc387a..c56f0a71c 100644
--- a/tests/paramgrill.c
+++ b/tests/paramgrill.c
@@ -53,6 +53,8 @@ static const int g_maxNbVariations = 64;
 *  Macros
 **************************************/
 #define DISPLAY(...)  fprintf(stderr, __VA_ARGS__)
+#define DISPLAYLEVEL(n, ...) if(g_displayLevel >= n) { fprintf(stderr, __VA_ARGS__); }
+
 #define TIMED 0
 #ifndef DEBUG
 #  define DEBUG 0
@@ -233,6 +235,7 @@ static U32 g_noSeed = 0;
 static paramValues_t g_params; /* Initialized at the beginning of main w/ emptyParams() function */
 static UTIL_time_t g_time; /* to be used to compare solution finding speeds to compare to original */
 static U32 g_memoTableLog = PARAM_UNSET;
+static U32 g_displayLevel = 3;
 
 typedef enum {
     directMap,
@@ -1282,7 +1285,6 @@ static void BMK_printWinnerOpt(FILE* f, const U32 cLevel, const BMK_result_t res
         /* the table */
         fprintf(f, "================================\n");
         for(n = g_winners; n != NULL; n = n->next) {
-            fprintf(f, "\r%79s\r", "");
             BMK_displayOneResult(f, n->res, srcSize);
         }
         fprintf(f, "================================\n");
@@ -2313,7 +2315,6 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_
                 }
 
                 BMK_printWinnerOpt(stdout, CUSTOM_LEVEL, winner.result, winner.params, target, buf.srcSize);
-                BMK_translateAdvancedParams(stdout, winner.params);
             }
         }
 
@@ -2364,7 +2365,6 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_
         }
         /* end summary */
         BMK_printWinnerOpt(stdout, CUSTOM_LEVEL, winner.result, winner.params, target, buf.srcSize);
-        BMK_translateAdvancedParams(stdout, winner.params);
         DISPLAY("grillParams size - optimizer completed \n");
 
     }
@@ -2653,6 +2653,14 @@ int main(int argc, const char** argv)
                     seperateFiles = 1;
                     break;
 
+                case 'q':
+                    g_displayLevel--;
+                    break;
+
+                case 'v':
+                    g_displayLevel++;
+                    break;
+
                 /* load dictionary file (only applicable for optimizer rn) */
                 case 'D':
                     if(i == argc - 1) { /* last argument, return error. */ 

From 4d9c6f51b8f34970fc292d82f23a2fa7f3bdede0 Mon Sep 17 00:00:00 2001
From: George Lu 
Date: Tue, 14 Aug 2018 15:54:07 -0700
Subject: [PATCH 169/372] -q -v options

---
 tests/paramgrill.c | 31 +++++++++++++++++--------------
 1 file changed, 17 insertions(+), 14 deletions(-)

diff --git a/tests/paramgrill.c b/tests/paramgrill.c
index c56f0a71c..ef6d6d46a 100644
--- a/tests/paramgrill.c
+++ b/tests/paramgrill.c
@@ -1257,22 +1257,22 @@ static void BMK_printWinnerOpt(FILE* f, const U32 cLevel, const BMK_result_t res
     /* global winner used for constraints */
                                     /* cSize, cSpeed, dSpeed, cMem */
     static winnerInfo_t g_winner = { { (size_t)-1LL, 0, 0, (size_t)-1LL }, { { PARAM_UNSET, PARAM_UNSET, PARAM_UNSET, PARAM_UNSET, PARAM_UNSET, PARAM_UNSET, PARAM_UNSET, PARAM_UNSET } } };
-    if(DEBUG || compareResultLT(g_winner.result, result, targetConstraints, srcSize)) {
+    if(DEBUG || compareResultLT(g_winner.result, result, targetConstraints, srcSize) || g_displayLevel >= 4) {
         if(DEBUG && compareResultLT(g_winner.result, result, targetConstraints, srcSize)) {
             DISPLAY("New Winner: \n");
         }
 
-        BMK_printWinner(f, cLevel, result, params, srcSize);
+        if(g_displayLevel >= 2) { BMK_printWinner(f, cLevel, result, params, srcSize); }
 
         if(compareResultLT(g_winner.result, result, targetConstraints, srcSize)) {
-            BMK_translateAdvancedParams(f, params);
+            if(g_displayLevel >= 1) { BMK_translateAdvancedParams(f, params); }
             g_winner.result = result;
             g_winner.params = params;
         }
     }  
 
     //prints out tradeoff table if using lvloptimize
-    if(g_optmode && g_optimizer) {
+    if(g_optmode && g_optimizer && (DEBUG || g_displayLevel == 3)) {
         winnerInfo_t w;
         winner_ll_node* n;
         w.result = result;
@@ -2205,9 +2205,9 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_
     }
 
     if(nbFiles == 1) {
-        DISPLAY("Loading %s...       \r", fileNamesTable[0]);
+        DISPLAYLEVEL(2, "Loading %s...       \r", fileNamesTable[0]);
     } else {
-        DISPLAY("Loading %lu Files...       \r", (unsigned long)nbFiles); 
+        DISPLAYLEVEL(2, "Loading %lu Files...       \r", (unsigned long)nbFiles); 
     }
 
     /* sanitize paramTarget */
@@ -2273,16 +2273,16 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_
     }
 
     /* bench */
-    DISPLAY("\r%79s\r", "");
+    DISPLAYLEVEL(2, "\r%79s\r", "");
     if(nbFiles == 1) {
-        DISPLAY("optimizing for %s", fileNamesTable[0]);
+        DISPLAYLEVEL(2, "optimizing for %s", fileNamesTable[0]);
     } else {
-        DISPLAY("optimizing for %lu Files", (unsigned long)nbFiles);
+        DISPLAYLEVEL(2, "optimizing for %lu Files", (unsigned long)nbFiles);
     }
 
-    if(target.cSpeed != 0) { DISPLAY(" - limit compression speed %u MB/s", target.cSpeed >> 20); }
-    if(target.dSpeed != 0) { DISPLAY(" - limit decompression speed %u MB/s", target.dSpeed >> 20); }
-    if(target.cMem != (U32)-1) { DISPLAY(" - limit memory %u MB", target.cMem >> 20); }
+    if(target.cSpeed != 0) { DISPLAYLEVEL(2," - limit compression speed %u MB/s", target.cSpeed >> 20); }
+    if(target.dSpeed != 0) { DISPLAYLEVEL(2, " - limit decompression speed %u MB/s", target.dSpeed >> 20); }
+    if(target.cMem != (U32)-1) { DISPLAYLEVEL(2, " - limit memory %u MB", target.cMem >> 20); }
 
     DISPLAY("\n");
     findClockGranularity();
@@ -2364,7 +2364,8 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_
             goto _cleanUp;
         }
         /* end summary */
-        BMK_printWinnerOpt(stdout, CUSTOM_LEVEL, winner.result, winner.params, target, buf.srcSize);
+        BMK_displayOneResult(stdout, winner, buf.srcSize);
+        BMK_translateAdvancedParams(stdout, winner.params);
         DISPLAY("grillParams size - optimizer completed \n");
 
     }
@@ -2501,7 +2502,7 @@ int main(int argc, const char** argv)
     assert(argc>=1);   /* for exename */
 
     /* Welcome message */
-    DISPLAY(WELCOME_MESSAGE);
+    DISPLAYLEVEL(2, WELCOME_MESSAGE);
 
     for(i=1; i
Date: Tue, 14 Aug 2018 16:51:39 -0700
Subject: [PATCH 170/372] Clean up repetitive display

Add documentation
---
 tests/README.md    |  2 ++
 tests/paramgrill.c | 28 +++++++++++++---------------
 2 files changed, 15 insertions(+), 15 deletions(-)

diff --git a/tests/README.md b/tests/README.md
index bdc9fff97..60fcebfbc 100644
--- a/tests/README.md
+++ b/tests/README.md
@@ -127,6 +127,8 @@ Full list of arguments
  -v           : Prints Benchmarking output
  -D           : Next argument dictionary file
  -s           : Benchmark all files separately
+ -q           : Quiet, repeat for more quiet
+ -v           : Verbose, cancels quiet, repeat for more volume
 
 ```
  Any inputs afterwards are treated as files to benchmark.
diff --git a/tests/paramgrill.c b/tests/paramgrill.c
index ef6d6d46a..ef005dd9a 100644
--- a/tests/paramgrill.c
+++ b/tests/paramgrill.c
@@ -235,7 +235,7 @@ static U32 g_noSeed = 0;
 static paramValues_t g_params; /* Initialized at the beginning of main w/ emptyParams() function */
 static UTIL_time_t g_time; /* to be used to compare solution finding speeds to compare to original */
 static U32 g_memoTableLog = PARAM_UNSET;
-static U32 g_displayLevel = 3;
+static int g_displayLevel = 3;
 
 typedef enum {
     directMap,
@@ -889,7 +889,7 @@ static int createBuffers(buffers_t* buff, const char* const * const fileNamesTab
             goto _cleanUp;
         }
 
-        DISPLAY("Loading %s...       \r", fileNamesTable[n]);
+        DISPLAYLEVEL(2, "Loading %s...       \r", fileNamesTable[n]);
 
         if (fileSize + pos > benchedSize) fileSize = benchedSize - pos, nbFiles=n;   /* buffer too small - stop after this file */
         {
@@ -2020,7 +2020,6 @@ static winnerInfo_t climbOnce(const constraint_t target,
             better = 0;
             DEBUGOUTPUT("Start\n");
             cparam = winnerInfo.params;
-            BMK_printWinnerOpt(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, target, buf.srcSize);
             candidateInfo.params = cparam;
              /* all dist-1 candidates */
             for(i = 0; i < varLen; i++) {
@@ -2036,7 +2035,6 @@ static winnerInfo_t climbOnce(const constraint_t target,
                         DEBUGOUTPUT("Res: %d\n", res);
                         if(res == BETTER_RESULT) { /* synonymous with better when called w/ infeasibleBM */
                             winnerInfo = candidateInfo;
-                            BMK_printWinnerOpt(stdout, CUSTOM_LEVEL, winnerInfo.result, sanitizeParams(winnerInfo.params), target, buf.srcSize);
                             better = 1;
                             if(compareResultLT(bestFeasible1.result, winnerInfo.result, target, buf.srcSize)) {
                                 bestFeasible1 = winnerInfo;
@@ -2063,7 +2061,6 @@ static winnerInfo_t climbOnce(const constraint_t target,
                     DEBUGOUTPUT("Res: %d\n", res);
                     if(res == BETTER_RESULT) { /* synonymous with better in this case*/
                         winnerInfo = candidateInfo;
-                        BMK_printWinnerOpt(stdout, CUSTOM_LEVEL, winnerInfo.result, sanitizeParams(winnerInfo.params), target, buf.srcSize);
                         better = 1;
                         if(compareResultLT(bestFeasible1.result, winnerInfo.result, target, buf.srcSize)) {
                             bestFeasible1 = winnerInfo;
@@ -2284,7 +2281,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_
     if(target.dSpeed != 0) { DISPLAYLEVEL(2, " - limit decompression speed %u MB/s", target.dSpeed >> 20); }
     if(target.cMem != (U32)-1) { DISPLAYLEVEL(2, " - limit memory %u MB", target.cMem >> 20); }
 
-    DISPLAY("\n");
+    DISPLAYLEVEL(2, "\n");
     findClockGranularity();
 
     {   
@@ -2309,7 +2306,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_
                         winner.params = CParams;
                     }
 
-                    CHECKTIMEGT(ret, 0, _cleanUp); /* if pass time limit, stop */
+                    CHECKTIMEGT(ret, 0, _displayCleanUp); /* if pass time limit, stop */
                     /* if the current params are too slow, just stop. */
                     if(target.cSpeed > candidate.cSpeed * 3 / 2) { break; }
                 }
@@ -2332,7 +2329,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_
                     if(compareResultLT(winner.result, w1.result, target, buf.srcSize)) {
                         winner = w1;
                     }
-                    CHECKTIMEGT(ret, 0, _cleanUp);
+                    CHECKTIMEGT(ret, 0, _displayCleanUp);
                 }
 
                 while(st && tries > 0) {
@@ -2349,7 +2346,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_
                         st = nextStrategy(st, bestStrategy);
                         tries -= TRY_DECAY;
                     }
-                    CHECKTIMEGT(ret, 0, _cleanUp);
+                    CHECKTIMEGT(ret, 0, _displayCleanUp);
                 }
             } else {
                 winner = optimizeFixedStrategy(buf, ctx, target, paramBase, paramTarget.vals[strt_ind], allMT, g_maxTries);
@@ -2364,12 +2361,13 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_
             goto _cleanUp;
         }
         /* end summary */
-        BMK_displayOneResult(stdout, winner, buf.srcSize);
+_displayCleanUp: 
+        if(g_displayLevel >= 0) { BMK_displayOneResult(stdout, winner, buf.srcSize); }
         BMK_translateAdvancedParams(stdout, winner.params);
-        DISPLAY("grillParams size - optimizer completed \n");
+        DISPLAYLEVEL(1, "grillParams size - optimizer completed \n");
 
     }
-_cleanUp: 
+_cleanUp:
     freeContexts(ctx);
     freeBuffers(buf);
     freeMemoTableArray(allMT);
@@ -2501,9 +2499,6 @@ int main(int argc, const char** argv)
 
     assert(argc>=1);   /* for exename */
 
-    /* Welcome message */
-    DISPLAYLEVEL(2, WELCOME_MESSAGE);
-
     for(i=1; i
Date: Tue, 14 Aug 2018 18:04:58 -0700
Subject: [PATCH 171/372] silencing params

---
 tests/README.md    |  1 +
 tests/paramgrill.c | 70 ++++++++++++++++++++++++++++++++++++----------
 2 files changed, 56 insertions(+), 15 deletions(-)

diff --git a/tests/README.md b/tests/README.md
index 60fcebfbc..3af08f7be 100644
--- a/tests/README.md
+++ b/tests/README.md
@@ -122,6 +122,7 @@ Full list of arguments
     tries=    : Maximum number of random restarts on a single strategy before switching (Default 3)
                 Higher values will make optimizer run longer, more chances to find better solution.
     memLog    : Limits the log of the size of each memotable (1 per strategy). Setting memLog = 0 turns off memoization 
+ --display=   : which params to display, uses all --zstd parameter names and 'cParams' to display only compression parameters. 
  -P#          : generated sample compressibility 
  -t#          : Caps runtime of operation in seconds (default : 99999 seconds (about 27 hours )) 
  -v           : Prints Benchmarking output
diff --git a/tests/paramgrill.c b/tests/paramgrill.c
index ef005dd9a..2f00c198c 100644
--- a/tests/paramgrill.c
+++ b/tests/paramgrill.c
@@ -237,6 +237,8 @@ static UTIL_time_t g_time; /* to be used to compare solution finding speeds to c
 static U32 g_memoTableLog = PARAM_UNSET;
 static int g_displayLevel = 3;
 
+static BYTE g_silenceParams[NUM_PARAMS];
+
 typedef enum {
     directMap,
     xxhashMap,
@@ -447,31 +449,34 @@ static paramValues_t cParamUnsetMin(paramValues_t paramTarget) {
 }
 
 static void BMK_translateAdvancedParams(FILE* f, const paramValues_t params) {
-    U32 i;
+    varInds_t v;
+    int first = 1;
     fprintf(f,"--zstd=");
-    for(i = 0; i < NUM_PARAMS; i++) {
-        fprintf(f,"%s=", g_paramNames[i]);
+    for(v = 0; v < NUM_PARAMS; v++) {
+        if(g_silenceParams[v]) { continue; }
+        if(!first) { fprintf(f, ","); }
+        fprintf(f,"%s=", g_paramNames[v]);
         
-        if(i == strt_ind) { fprintf(f,"%u", params.vals[i]); }
-        else { displayParamVal(f, i, params.vals[i], 0); }
-
-        if(i != NUM_PARAMS - 1) {
-            fprintf(f, ",");
-        }
+        if(v == strt_ind) { fprintf(f,"%u", params.vals[v]); }
+        else { displayParamVal(f, v, params.vals[v], 0); }
+        first = 0;
     }
     fprintf(f, "\n");
 }
 
 static void BMK_displayOneResult(FILE* f, winnerInfo_t res, const size_t srcSize) {
             varInds_t v;
+            int first = 1;
             res.params = cParamUnsetMin(res.params);
             fprintf(f,"    {");
             for(v = 0; v < NUM_PARAMS; v++) {
-                if(v != 0) { fprintf(f, ","); }
+                if(g_silenceParams[v]) { continue; }
+                if(!first) { fprintf(f, ","); }
                 displayParamVal(f, v, res.params.vals[v], 3);
+                first = 0;
             }
 
-            fprintf(f, "},     /* R:%5.3f at %5.1f MB/s - %5.1f MB/s */\n",
+            fprintf(f, " },     /* R:%5.3f at %5.1f MB/s - %5.1f MB/s */\n",
             (double)srcSize / res.result.cSize, (double)res.result.cSpeed / (1 MB), (double)res.result.dSpeed / (1 MB));
 }
 
@@ -2545,6 +2550,43 @@ int main(int argc, const char** argv)
             }
             continue;
             /* if not return, success */
+
+        } else if (longCommandWArg(&argument, "--display=")) {
+            /* Decode command (note : aggregated commands are allowed) */
+            memset(g_silenceParams, 1, sizeof(g_silenceParams));
+            for ( ; ;) {
+                int found = 0;
+                varInds_t v;
+                for(v = 0; v < NUM_PARAMS; v++) {
+                    if(longCommandWArg(&argument, g_shortParamNames[v]) || longCommandWArg(&argument, g_paramNames[v])) {
+                        g_silenceParams[v] = 0;
+                        found = 1;
+                    }
+                }
+                if(longCommandWArg(&argument, "compressionParameters") || longCommandWArg(&argument, "cParams")) {
+                    for(v = 0; v <= strt_ind; v++) {
+                        g_silenceParams[v] = 0;
+                    }
+                    found = 1;
+                }
+
+
+                if(found) {
+                    if(argument[0]==',') {
+                        continue;
+                    } else {
+                        break;
+                    }
+                }
+                DISPLAY("invalid parameter name parameter \n");
+                return 1;
+            }
+
+            if (argument[0] != 0) {
+                DISPLAY("invalid --display format\n");
+                return 1; /* check the end of string */
+            }
+            continue;
         } else if (argument[0]=='-') {
             argument++;
 
@@ -2650,13 +2692,11 @@ int main(int argc, const char** argv)
                     break;
 
                 case 'q':
-                    argument++;
-                    g_displayLevel--;
+                    while (argument[0] == 'q') { argument++; g_displayLevel--; }
                     break;
 
                 case 'v':
-                    argument++;
-                    g_displayLevel++;
+                    while (argument[0] == 'v') { argument++; g_displayLevel++; }
                     break;
 
                 /* load dictionary file (only applicable for optimizer rn) */

From ee77ddc28ddb0ff9bae63ee61e58a3ecfb315728 Mon Sep 17 00:00:00 2001
From: George Lu 
Date: Wed, 15 Aug 2018 11:46:19 -0700
Subject: [PATCH 172/372] Fix wraparound

---
 tests/paramgrill.c | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/tests/paramgrill.c b/tests/paramgrill.c
index 2f00c198c..0d814bf7c 100644
--- a/tests/paramgrill.c
+++ b/tests/paramgrill.c
@@ -144,8 +144,8 @@ static const char* g_shortParamNames[NUM_PARAMS] =
         { "wlog", "clog", "hlog","slog", "slen", "tlen", "strt", "fadt" };
 
 /* maps value from { 0 to rangetable[param] - 1 } to valid paramvalues */
-static U32 rangeMap(varInds_t param, U32 ind) {
-    ind = MIN(ind, rangetable[param] - 1);
+static U32 rangeMap(varInds_t param, int ind) {
+    ind = MAX(MIN(ind, (int)rangetable[param] - 1), 0);
     switch(param) {
         case tlen_ind:
             return tlen_table[ind];
@@ -166,7 +166,7 @@ static U32 rangeMap(varInds_t param, U32 ind) {
 }
 
 /* inverse of rangeMap */
-static U32 invRangeMap(varInds_t param, U32 value) {
+static int invRangeMap(varInds_t param, U32 value) {
     value = MIN(MAX(mintable[param], value), maxtable[param]);
     switch(param) {
         case tlen_ind: /* bin search */
@@ -186,7 +186,7 @@ static U32 invRangeMap(varInds_t param, U32 value) {
             return lo;    
         }
         case fadt_ind:
-            return value + 1;
+            return (int)value + 1;
         case wlog_ind:
         case clog_ind:
         case hlog_ind:
@@ -196,7 +196,7 @@ static U32 invRangeMap(varInds_t param, U32 value) {
             return value - mintable[param];
         case NUM_PARAMS:
             DISPLAY("Error, not a valid param\n ");
-            return (U32)-1;
+            return -2;
     }
     return 0; /* should never happen, stop compiler warnings */
 }
@@ -1545,7 +1545,7 @@ static unsigned memoTableIndDirect(const paramValues_t* ptr, const varInds_t* va
     for(i = 0; i < varyLen; i++) {
         varInds_t v = varyParams[i];
         if(v == strt_ind) continue; /* exclude strategy from memotable */
-        ind *= rangetable[v]; ind += invRangeMap(v, ptr->vals[v]);
+        ind *= rangetable[v]; ind += (unsigned)invRangeMap(v, ptr->vals[v]);
     }
     return ind;
 }

From b234870c33745401138af634909acf46ec3363fc Mon Sep 17 00:00:00 2001
From: George Lu 
Date: Wed, 15 Aug 2018 14:29:49 -0700
Subject: [PATCH 173/372] clarify display README

---
 tests/README.md | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/tests/README.md b/tests/README.md
index 3af08f7be..442285dbc 100644
--- a/tests/README.md
+++ b/tests/README.md
@@ -122,7 +122,10 @@ Full list of arguments
     tries=    : Maximum number of random restarts on a single strategy before switching (Default 3)
                 Higher values will make optimizer run longer, more chances to find better solution.
     memLog    : Limits the log of the size of each memotable (1 per strategy). Setting memLog = 0 turns off memoization 
- --display=   : which params to display, uses all --zstd parameter names and 'cParams' to display only compression parameters. 
+ --display=   : specifiy which parameters are included in the output
+                can use all --zstd parameter names and 'cParams' as a shorthand for all parameters used in ZSTD_compressionParameters 
+                (Default: display all params available)
+
  -P#          : generated sample compressibility 
  -t#          : Caps runtime of operation in seconds (default : 99999 seconds (about 27 hours )) 
  -v           : Prints Benchmarking output

From 42a02ab745976068255fe1a43915b70b8f9aa104 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Wed, 15 Aug 2018 14:35:38 -0700
Subject: [PATCH 174/372] fixed minor warnings issued by scan-build

---
 Makefile                                      | 4 ++--
 contrib/seekable_format/zstdseek_decompress.c | 5 ++++-
 lib/decompress/zstd_decompress.c              | 1 +
 programs/dibio.c                              | 6 +++++-
 programs/dibio.h                              | 2 +-
 5 files changed, 13 insertions(+), 5 deletions(-)

diff --git a/Makefile b/Makefile
index 6c4ab9c9b..f8441e60d 100644
--- a/Makefile
+++ b/Makefile
@@ -114,7 +114,7 @@ clean:
 ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD DragonFly NetBSD MSYS_NT))
 
 HOST_OS = POSIX
-CMAKE_PARAMS = -DZSTD_BUILD_CONTRIB:BOOL=ON -DZSTD_BUILD_STATIC:BOOL=ON -DZSTD_BUILD_TESTS:BOOL=ON -DZSTD_ZLIB_SUPPORT:BOOL=ON -DZSTD_LZMA_SUPPORT:BOOL=ON -DCMAKE_BUILD_TYPE=Release 
+CMAKE_PARAMS = -DZSTD_BUILD_CONTRIB:BOOL=ON -DZSTD_BUILD_STATIC:BOOL=ON -DZSTD_BUILD_TESTS:BOOL=ON -DZSTD_ZLIB_SUPPORT:BOOL=ON -DZSTD_LZMA_SUPPORT:BOOL=ON -DCMAKE_BUILD_TYPE=Release
 
 .PHONY: list
 list:
@@ -351,7 +351,7 @@ bmi32build: clean
 	$(CC) -v
 	CFLAGS="-O3 -mbmi -m32 -Werror" $(MAKE) -C $(TESTDIR) test
 
-staticAnalyze: clean
+staticAnalyze:
 	$(CC) -v
 	CPPFLAGS=-g scan-build --status-bugs -v $(MAKE) all
 endif
diff --git a/contrib/seekable_format/zstdseek_decompress.c b/contrib/seekable_format/zstdseek_decompress.c
index b006ff834..2b109b9d0 100644
--- a/contrib/seekable_format/zstdseek_decompress.c
+++ b/contrib/seekable_format/zstdseek_decompress.c
@@ -56,6 +56,7 @@
 
 #include  /* malloc, free */
 #include   /* FILE* */
+#include 
 
 #define XXH_STATIC_LINKING_ONLY
 #define XXH_NAMESPACE ZSTD_
@@ -112,7 +113,7 @@ static int ZSTD_seekable_read_buff(void* opaque, void* buffer, size_t n)
 
 static int ZSTD_seekable_seek_buff(void* opaque, long long offset, int origin)
 {
-    buffWrapper_t* buff = (buffWrapper_t*) opaque;
+    buffWrapper_t* const buff = (buffWrapper_t*) opaque;
     unsigned long long newOffset;
     switch (origin) {
     case SEEK_SET:
@@ -124,6 +125,8 @@ static int ZSTD_seekable_seek_buff(void* opaque, long long offset, int origin)
     case SEEK_END:
         newOffset = (unsigned long long)buff->size - offset;
         break;
+    default:
+        assert(0);  /* not possible */
     }
     if (newOffset > buff->size) {
         return -1;
diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c
index 16a08fb6b..5d9f0ba0b 100644
--- a/lib/decompress/zstd_decompress.c
+++ b/lib/decompress/zstd_decompress.c
@@ -330,6 +330,7 @@ size_t ZSTD_getFrameHeader_advanced(ZSTD_frameHeader* zfhPtr, const void* src, s
     const BYTE* ip = (const BYTE*)src;
     size_t const minInputSize = ZSTD_startingInputLength(format);
 
+    memset(zfhPtr, 0, sizeof(*zfhPtr));   /* not strictly necessary, but static analyzer do not understand that zfhPtr is only going to be read only if return value is zero, since they are 2 different signals */
     if (srcSize < minInputSize) return minInputSize;
     if (src==NULL) return ERROR(GENERIC);   /* invalid parameter */
 
diff --git a/programs/dibio.c b/programs/dibio.c
index 5d1f6d6c4..fbb8aa6fa 100644
--- a/programs/dibio.c
+++ b/programs/dibio.c
@@ -27,6 +27,7 @@
 #include          /* memset */
 #include           /* fprintf, fopen, ftello64 */
 #include           /* errno */
+#include 
 
 #include "mem.h"            /* read */
 #include "error_private.h"
@@ -165,6 +166,7 @@ static U32 DiB_rand(U32* src)
 static void DiB_shuffle(const char** fileNamesTable, unsigned nbFiles) {
     U32 seed = 0xFD2FB528;
     unsigned i;
+    assert(nbFiles >= 1);
     for (i = nbFiles - 1; i > 0; --i) {
         unsigned const j = DiB_rand(&seed) % (i + 1);
         const char* const tmp = fileNamesTable[j];
@@ -310,7 +312,7 @@ int DiB_trainFromFiles(const char* dictFileName, unsigned maxDictSize,
     /* Load input buffer */
     DISPLAYLEVEL(3, "Shuffling input files\n");
     DiB_shuffle(fileNamesTable, nbFiles);
-    nbFiles = DiB_loadFiles(srcBuffer, &loadedSize, sampleSizes, fs.nbSamples, fileNamesTable, nbFiles, chunkSize, displayLevel);
+    DiB_loadFiles(srcBuffer, &loadedSize, sampleSizes, fs.nbSamples, fileNamesTable, nbFiles, chunkSize, displayLevel);
 
     {   size_t dictSize;
         if (params) {
@@ -319,6 +321,7 @@ int DiB_trainFromFiles(const char* dictFileName, unsigned maxDictSize,
                                                            srcBuffer, sampleSizes, fs.nbSamples,
                                                            *params);
         } else if (optimizeCover) {
+            assert(coverParams != NULL);
             dictSize = ZDICT_optimizeTrainFromBuffer_cover(dictBuffer, maxDictSize,
                                                            srcBuffer, sampleSizes, fs.nbSamples,
                                                            coverParams);
@@ -327,6 +330,7 @@ int DiB_trainFromFiles(const char* dictFileName, unsigned maxDictSize,
                 DISPLAYLEVEL(2, "k=%u\nd=%u\nsteps=%u\nsplit=%u\n", coverParams->k, coverParams->d, coverParams->steps, splitPercentage);
             }
         } else {
+            assert(coverParams != NULL);
             dictSize = ZDICT_trainFromBuffer_cover(dictBuffer, maxDictSize, srcBuffer,
                                                    sampleSizes, fs.nbSamples, *coverParams);
         }
diff --git a/programs/dibio.h b/programs/dibio.h
index 499e30365..31a6b4bdb 100644
--- a/programs/dibio.h
+++ b/programs/dibio.h
@@ -33,7 +33,7 @@
 */
 int DiB_trainFromFiles(const char* dictFileName, unsigned maxDictSize,
                        const char** fileNamesTable, unsigned nbFiles, size_t chunkSize,
-                       ZDICT_legacy_params_t *params, ZDICT_cover_params_t *coverParams,
+                       ZDICT_legacy_params_t* params, ZDICT_cover_params_t* coverParams,
                        int optimizeCover);
 
 #endif

From 46be2ef5d8f48b6509e6813f74f3622db0fbc09f Mon Sep 17 00:00:00 2001
From: George Lu 
Date: Wed, 15 Aug 2018 14:00:57 -0700
Subject: [PATCH 175/372] Remove unused stuff

---
 tests/README.md    |   3 +-
 tests/paramgrill.c | 164 ++++++++++++++++++++-------------------------
 2 files changed, 73 insertions(+), 94 deletions(-)

diff --git a/tests/README.md b/tests/README.md
index 442285dbc..c1128571a 100644
--- a/tests/README.md
+++ b/tests/README.md
@@ -95,7 +95,6 @@ Full list of arguments
 ```
  -T#          : set level 1 speed objective
  -B#          : cut input into blocks of size # (default : single block)
- -i#          : iteration loops
  -S           : benchmarks a single run (example command: -Sl3w10h12)
     w# - windowLog
     h# - hashLog
@@ -119,7 +118,7 @@ Full list of arguments
     speedRatio=   (accepts decimals)
               : determines value of gains in speed vs gains in ratio
                 when determining overall winner (default 5 (1% ratio = 5% speed)).
-    tries=    : Maximum number of random restarts on a single strategy before switching (Default 3)
+    tries=    : Maximum number of random restarts on a single strategy before switching (Default 5)
                 Higher values will make optimizer run longer, more chances to find better solution.
     memLog    : Limits the log of the size of each memotable (1 per strategy). Setting memLog = 0 turns off memoization 
  --display=   : specifiy which parameters are included in the output
diff --git a/tests/paramgrill.c b/tests/paramgrill.c
index 0d814bf7c..e5e1e2a36 100644
--- a/tests/paramgrill.c
+++ b/tests/paramgrill.c
@@ -37,9 +37,6 @@
 #define WELCOME_MESSAGE "*** %s %s %i-bits, by %s ***\n", PROGRAM_DESCRIPTION, ZSTD_VERSION_STRING, (int)(sizeof(void*)*8), AUTHOR
 
 #define TIMELOOP_NANOSEC      (1*1000000000ULL) /* 1 second */
-
-#define NBLOOPS    2
-#define TIMELOOP  (2 * SEC_TO_MICRO)
 #define NB_LEVELS_TRACKED 22   /* ensured being >= ZSTD_maxCLevel() in BMK_init_level_constraints() */
 
 static const size_t maxMemory = (sizeof(size_t)==4)  ?  (2 GB - 64 MB) : (size_t)(1ULL << ((sizeof(size_t)*8)-31));
@@ -54,12 +51,12 @@ static const int g_maxNbVariations = 64;
 **************************************/
 #define DISPLAY(...)  fprintf(stderr, __VA_ARGS__)
 #define DISPLAYLEVEL(n, ...) if(g_displayLevel >= n) { fprintf(stderr, __VA_ARGS__); }
+#define DEBUGOUTPUT(...) { if (DEBUG) DISPLAY(__VA_ARGS__); }
 
 #define TIMED 0
 #ifndef DEBUG
 #  define DEBUG 0
 #endif
-#define DEBUGOUTPUT(...) { if (DEBUG) DISPLAY(__VA_ARGS__); }
 
 #undef MIN
 #undef MAX
@@ -223,21 +220,30 @@ static void displayParamVal(FILE* f, varInds_t param, U32 value, int width) {
 
 typedef BYTE U8;
 
+/* General Utility */
 static U32 g_timeLimit_s = 99999;   /* about 27 hours */
-static U32 g_nbIterations = NBLOOPS;
-static double g_compressibility = COMPRESSIBILITY_DEFAULT;
+static UTIL_time_t g_time; /* to be used to compare solution finding speeds to compare to original */
 static U32 g_blockSize = 0;
 static U32 g_rand = 1;
+
+/* Display */
+static int g_displayLevel = 3;
+static BYTE g_silenceParams[NUM_PARAMS];
+
+/* Mode Selection */
 static U32 g_singleRun = 0;
 static U32 g_optimizer = 0;
+static int g_optmode = 0;
+
+/* For cLevel Table generation */
 static U32 g_target = 0;
 static U32 g_noSeed = 0;
-static paramValues_t g_params; /* Initialized at the beginning of main w/ emptyParams() function */
-static UTIL_time_t g_time; /* to be used to compare solution finding speeds to compare to original */
-static U32 g_memoTableLog = PARAM_UNSET;
-static int g_displayLevel = 3;
 
-static BYTE g_silenceParams[NUM_PARAMS];
+/* For optimizer */
+static paramValues_t g_params; /* Initialized at the beginning of main w/ emptyParams() function */
+static double g_ratioMultiplier = 5.;
+static U32 g_strictness = PARAM_UNSET; /* range 0 - 99, measure of how strict  */
+static BMK_result_t g_lvltarget;
 
 typedef enum {
     directMap,
@@ -258,11 +264,6 @@ typedef struct {
     paramValues_t params;
 } winnerInfo_t;
 
-typedef struct {
-    BMK_result_t result;
-    ZSTD_compressionParameters params;
-} oldWinnerInfo_t;
-
 typedef struct {
     U32 cSpeed;  /* bytes / sec */
     U32 dSpeed;
@@ -276,27 +277,13 @@ struct winner_ll_node {
 };
 
 static winner_ll_node* g_winners; /* linked list sorted ascending by cSize & cSpeed */
-static BMK_result_t g_lvltarget;
-static int g_optmode = 0;
-
-static double g_ratioMultiplier = 5.;
-
-/* g_mode? */
-
-/* range 0 - 99, measure of how strict  */
-static U32 g_strictness = PARAM_UNSET;
-
-void BMK_SetNbIterations(int nbLoops)
-{
-    g_nbIterations = nbLoops;
-    DISPLAY("- %u iterations -\n", g_nbIterations);
-}
 
 /*
  * Additional Global Variables (Defined Above Use)
  * g_level_constraint
  * g_alreadyTested
  * g_maxTries
+ * g_clockGranularity
  */
 
 /*-*******************************************************
@@ -340,7 +327,7 @@ static paramValues_t adjustParams(paramValues_t p, const size_t maxBlockSize, co
     paramValues_t ot = p;
     varInds_t i;
     p = cParamsToPVals(ZSTD_adjustCParams(pvalsToCParams(p), maxBlockSize, dictSize));
-
+    if(!dictSize) { p.vals[fadt_ind] = 0; }
     /* retain value of all other parameters */
     for(i = strt_ind + 1; i < NUM_PARAMS; i++) {
         p.vals[i] = ot.vals[i];
@@ -1306,7 +1293,7 @@ static void BMK_printWinnerOpt(FILE* f, const U32 cLevel, const BMK_result_t res
     }
 }
 
-static void BMK_printWinners2(FILE* f, const oldWinnerInfo_t* winners, const size_t srcSize)
+static void BMK_printWinners2(FILE* f, const winnerInfo_t* winners, const size_t srcSize)
 {
     int cLevel;
 
@@ -1314,11 +1301,11 @@ static void BMK_printWinners2(FILE* f, const oldWinnerInfo_t* winners, const siz
     fprintf(f, "    /* W,  C,  H,  S,  L,  T, strat */ \n");
 
     for (cLevel=0; cLevel <= NB_LEVELS_TRACKED; cLevel++)
-        BMK_printWinner(f, cLevel, winners[cLevel].result, cParamsToPVals(winners[cLevel].params), srcSize);
+        BMK_printWinner(f, cLevel, winners[cLevel].result, winners[cLevel].params, srcSize);
 }
 
 
-static void BMK_printWinners(FILE* f, const oldWinnerInfo_t* winners, const size_t srcSize)
+static void BMK_printWinners(FILE* f, const winnerInfo_t* winners, const size_t srcSize)
 {
     fseek(f, 0, SEEK_SET);
     BMK_printWinners2(f, winners, srcSize);
@@ -1355,14 +1342,14 @@ static void BMK_init_level_constraints(int bytePerSec_level1)
     }   }
 }
 
-static int BMK_seed(oldWinnerInfo_t* winners, const ZSTD_compressionParameters params, 
+static int BMK_seed(winnerInfo_t* winners, const paramValues_t params, 
                     const buffers_t buf, const contexts_t ctx)
 {
     BMK_result_t testResult;
     int better = 0;
     int cLevel;
 
-    BMK_benchParam(&testResult, buf, ctx, cParamsToPVals(params));
+    BMK_benchParam(&testResult, buf, ctx, params);
 
 
     for (cLevel = 1; cLevel <= NB_LEVELS_TRACKED; cLevel++) {
@@ -1370,15 +1357,15 @@ static int BMK_seed(oldWinnerInfo_t* winners, const ZSTD_compressionParameters p
             continue;   /* not fast enough for this level */
         if (testResult.dSpeed < g_level_constraint[cLevel].dSpeed_min)
             continue;   /* not fast enough for this level */
-        if (params.windowLog > g_level_constraint[cLevel].windowLog_max)
+        if (params.vals[wlog_ind] > g_level_constraint[cLevel].windowLog_max)
             continue;   /* too much memory for this level */
-        if (params.strategy > g_level_constraint[cLevel].strategy_max)
+        if (params.vals[strt_ind] > g_level_constraint[cLevel].strategy_max)
             continue;   /* forbidden strategy for this level */
         if (winners[cLevel].result.cSize==0) {
             /* first solution for this cLevel */
             winners[cLevel].result = testResult;
             winners[cLevel].params = params;
-            BMK_printWinner(stdout, cLevel, testResult, cParamsToPVals(params), buf.srcSize);
+            BMK_printWinner(stdout, cLevel, testResult, params, buf.srcSize);
             better = 1;
             continue;
         }
@@ -1389,13 +1376,13 @@ static int BMK_seed(oldWinnerInfo_t* winners, const ZSTD_compressionParameters p
             double O_ratio = (double)buf.srcSize / winners[cLevel].result.cSize;
             double W_ratioNote = log (W_ratio);
             double O_ratioNote = log (O_ratio);
-            size_t W_DMemUsed = (1 << params.windowLog) + (16 KB);
-            size_t O_DMemUsed = (1 << winners[cLevel].params.windowLog) + (16 KB);
+            size_t W_DMemUsed = (1 << params.vals[wlog_ind]) + (16 KB);
+            size_t O_DMemUsed = (1 << winners[cLevel].params.vals[wlog_ind]) + (16 KB);
             double W_DMemUsed_note = W_ratioNote * ( 40 + 9*cLevel) - log((double)W_DMemUsed);
             double O_DMemUsed_note = O_ratioNote * ( 40 + 9*cLevel) - log((double)O_DMemUsed);
 
-            size_t W_CMemUsed = (1 << params.windowLog) + ZSTD_estimateCCtxSize_usingCParams(params);
-            size_t O_CMemUsed = (1 << winners[cLevel].params.windowLog) + ZSTD_estimateCCtxSize_usingCParams(winners[cLevel].params);
+            size_t W_CMemUsed = (1 << params.vals[wlog_ind]) + ZSTD_estimateCCtxSize_usingCParams(pvalsToCParams(params));
+            size_t O_CMemUsed = (1 << winners[cLevel].params.vals[wlog_ind]) + ZSTD_estimateCCtxSize_usingCParams(pvalsToCParams(winners[cLevel].params));
             double W_CMemUsed_note = W_ratioNote * ( 50 + 13*cLevel) - log((double)W_CMemUsed);
             double O_CMemUsed_note = O_ratioNote * ( 50 + 13*cLevel) - log((double)O_CMemUsed);
 
@@ -1443,7 +1430,7 @@ static int BMK_seed(oldWinnerInfo_t* winners, const ZSTD_compressionParameters p
 
             winners[cLevel].result = testResult;
             winners[cLevel].params = params;
-            BMK_printWinner(stdout, cLevel, testResult, cParamsToPVals(params), buf.srcSize);
+            BMK_printWinner(stdout, cLevel, testResult, params, buf.srcSize);
 
             better = 1;
     }   }
@@ -1587,7 +1574,7 @@ static void freeMemoTableArray(memoTable_t* const mtAll) {
 
 /* inits memotables for all (including mallocs), all strategies */
 /* takes unsanitized varyParams */
-static memoTable_t* createMemoTableArray(const varInds_t* const varyParams, const size_t varyLen) {
+static memoTable_t* createMemoTableArray(const varInds_t* const varyParams, const size_t varyLen, U32 memoTableLog) {
     memoTable_t* mtAll = (memoTable_t*)calloc(sizeof(memoTable_t),(ZSTD_btultra + 1));
     int i;
     
@@ -1600,7 +1587,7 @@ static memoTable_t* createMemoTableArray(const varInds_t* const varyParams, cons
     }
 
     /* no memoization */
-    if(g_memoTableLog == 0) {
+    if(memoTableLog == 0) {
         for(i = 1; i <= (int)ZSTD_btultra; i++) {
             mtAll[i].tableType = noMemo;
             mtAll[i].table = NULL;
@@ -1614,9 +1601,9 @@ static memoTable_t* createMemoTableArray(const varInds_t* const varyParams, cons
         size_t mtl = memoTableLen(mtAll[i].varArray, mtAll[i].varLen);
         mtAll[i].tableType = directMap;
 
-        if(g_memoTableLog != PARAM_UNSET && mtl > (1ULL << g_memoTableLog)) { /* use hash table */ /* provide some option to only use hash tables? */
+        if(memoTableLog != PARAM_UNSET && mtl > (1ULL << memoTableLog)) { /* use hash table */ /* provide some option to only use hash tables? */
             mtAll[i].tableType = xxhashMap;
-            mtl = (1ULL << g_memoTableLog);
+            mtl = (1ULL << memoTableLog);
         }
 
         mtAll[i].table = (BYTE*)calloc(sizeof(BYTE), mtl);
@@ -1646,21 +1633,19 @@ static paramValues_t overwriteParams(paramValues_t base, const paramValues_t mas
 #define PARAMTABLEMASK (PARAMTABLESIZE-1)
 static BYTE g_alreadyTested[PARAMTABLESIZE] = {0};   /* init to zero */
 
-static BYTE* NB_TESTS_PLAYED(ZSTD_compressionParameters p) {
-    ZSTD_compressionParameters p2 = pvalsToCParams(sanitizeParams(cParamsToPVals(p)));
+static BYTE* NB_TESTS_PLAYED(paramValues_t p) {
+    ZSTD_compressionParameters p2 = pvalsToCParams(sanitizeParams(p));
     return &g_alreadyTested[(XXH64((void*)&p2, sizeof(p2), 0) >> 3) & PARAMTABLEMASK];
 }
 
-static void playAround(FILE* f, oldWinnerInfo_t* winners,
-                       ZSTD_compressionParameters params,
+static void playAround(FILE* f, winnerInfo_t* winners,
+                       paramValues_t p,
                        const buffers_t buf, const contexts_t ctx)
 {
     int nbVariations = 0, i;
     UTIL_time_t const clockStart = UTIL_getTime();
 
     while (UTIL_clockSpanMicro(clockStart) < g_maxVariationTime) {
-        paramValues_t p = cParamsToPVals(params);
-        ZSTD_compressionParameters p2;
         BYTE* b;
 
         if (nbVariations++ > g_maxNbVariations) break;
@@ -1668,20 +1653,18 @@ static void playAround(FILE* f, oldWinnerInfo_t* winners,
         do { for(i = 0; i < 4; i++) { paramVaryOnce(FUZ_rand(&g_rand) % (strt_ind + 1), ((FUZ_rand(&g_rand) & 1) << 1) - 1, &p); } } 
         while(!paramValid(p));
 
-        p2 = pvalsToCParams(p);
-
         /* exclude faster if already played params */
-        if (FUZ_rand(&g_rand) & ((1 << *NB_TESTS_PLAYED(p2))-1))
+        if (FUZ_rand(&g_rand) & ((1 << *NB_TESTS_PLAYED(p))-1))
             continue;
 
         /* test */
-        b = NB_TESTS_PLAYED(p2);
+        b = NB_TESTS_PLAYED(p);
         (*b)++;
-        if (!BMK_seed(winners, p2, buf, ctx)) continue;
+        if (!BMK_seed(winners, p, buf, ctx)) continue;
 
         /* improvement found => search more */
         BMK_printWinners(f, winners, buf.srcSize);
-        playAround(f, winners, p2, buf, ctx);
+        playAround(f, winners, p, buf, ctx);
     }
 
 }
@@ -1693,7 +1676,7 @@ static void randomConstrainedParams(paramValues_t* pc, const memoTable_t* memoTa
     size_t j;
     const memoTable_t mt = memoTableArray[st];
     pc->vals[strt_ind] = st;
-    for(j = 0; j < MIN(1ULL << g_memoTableLog, memoTableLen(mt.varArray, mt.varLen)); j++) {
+    for(j = 0; j < mt.tableLen; j++) {
         int i;
         for(i = 0; i < NUM_PARAMS; i++) {
             varInds_t v = mt.varArray[i];
@@ -1706,23 +1689,24 @@ static void randomConstrainedParams(paramValues_t* pc, const memoTable_t* memoTa
 }
 
 /* Completely random parameter selection */
-static ZSTD_compressionParameters randomParams(void)
+static paramValues_t randomParams(void)
 {
-    paramValues_t p; varInds_t v;
-    for(v = 0; v < NUM_PARAMS; v++) {
+    varInds_t v; paramValues_t p;
+    for(v = 0; v <= NUM_PARAMS; v++) {
         p.vals[v] = rangeMap(v, FUZ_rand(&g_rand) % rangetable[v]);
     }
-    return pvalsToCParams(p);
+    return p;
 }
 
 static void BMK_selectRandomStart(
-                       FILE* f, oldWinnerInfo_t* winners,
+                       FILE* f, winnerInfo_t* winners,
                        const buffers_t buf, const contexts_t ctx)
 {
     U32 const id = FUZ_rand(&g_rand) % (NB_LEVELS_TRACKED+1);
-    if ((id==0) || (winners[id].params.windowLog==0)) {
+    if ((id==0) || (winners[id].params.vals[wlog_ind]==0)) {
         /* use some random entry */
-        ZSTD_compressionParameters const p = ZSTD_adjustCParams(randomParams(), buf.srcSize, 0);
+        paramValues_t const p = adjustParams(cParamsToPVals(pvalsToCParams(randomParams())), /* defaults nonCompression parameters */
+            buf.srcSize, 0);
         playAround(f, winners, p, buf, ctx);
     } else {
         playAround(f, winners, winners[id].params, buf, ctx);
@@ -1731,8 +1715,8 @@ static void BMK_selectRandomStart(
 
 static void BMK_benchFullTable(const buffers_t buf, const contexts_t ctx) 
 {
-    ZSTD_compressionParameters params;
-    oldWinnerInfo_t winners[NB_LEVELS_TRACKED+1];
+    paramValues_t params;
+    winnerInfo_t winners[NB_LEVELS_TRACKED+1];
     const char* const rfName = "grillResults.txt";
     FILE* const f = fopen(rfName, "w");
 
@@ -1745,9 +1729,9 @@ static void BMK_benchFullTable(const buffers_t buf, const contexts_t ctx)
         BMK_init_level_constraints(g_target * (1 MB));
     } else {
         /* baseline config for level 1 */
-        ZSTD_compressionParameters const l1params = ZSTD_getCParams(1, buf.maxBlockSize, ctx.dictSize);
+        paramValues_t const l1params = cParamsToPVals(ZSTD_getCParams(1, buf.maxBlockSize, ctx.dictSize));
         BMK_result_t testResult;
-        BMK_benchParam(&testResult, buf, ctx, cParamsToPVals(l1params));
+        BMK_benchParam(&testResult, buf, ctx, l1params);
         BMK_init_level_constraints((int)((testResult.cSpeed * 31) / 32));
     }
 
@@ -1755,7 +1739,7 @@ static void BMK_benchFullTable(const buffers_t buf, const contexts_t ctx)
     {   const int maxSeeds = g_noSeed ? 1 : ZSTD_maxCLevel();
         int i;
         for (i=0; i<=maxSeeds; i++) {
-            params = ZSTD_getCParams(i, buf.maxBlockSize, 0);
+            params = cParamsToPVals(ZSTD_getCParams(i, buf.maxBlockSize, 0));
             BMK_seed(winners, params, buf, ctx);
     }   }
     BMK_printWinners(f, winners, buf.srcSize);
@@ -1788,7 +1772,7 @@ static int benchOnce(const buffers_t buf, const contexts_t ctx) {
     return 0;
 }
 
-static int benchSample(void)
+static int benchSample(double compressibility)
 {
     const char* const name = "Sample 10MB";
     size_t const benchedSize = 10 MB;
@@ -1803,7 +1787,7 @@ static int benchSample(void)
         return 2;
     }
 
-    RDG_genBuffer(srcBuffer, benchedSize, g_compressibility, 0.0, 0);
+    RDG_genBuffer(srcBuffer, benchedSize, compressibility, 0.0, 0);
 
     if(createBuffersFromMemory(&buf, srcBuffer, 1, &benchedSize)) {
         DISPLAY("Buffer Creation Error\n");
@@ -1819,7 +1803,7 @@ static int benchSample(void)
 
     /* bench */
     DISPLAY("\r%79s\r", "");
-    DISPLAY("using %s %i%%: \n", name, (int)(g_compressibility*100));
+    DISPLAY("using %s %i%%: \n", name, (int)(compressibility*100));
 
     if(g_singleRun) {
         ret = benchOnce(buf, ctx);
@@ -2180,10 +2164,11 @@ static int nextStrategy(const int currentStrategy, const int bestStrategy) {
  * cLevel - compression level to exceed (all solutions must be > lvl in cSpeed + ratio)
  */
 
-static int g_maxTries = 3;
+static int g_maxTries = 5;
 #define TRY_DECAY 1
 
-static int optimizeForSize(const char* const * const fileNamesTable, const size_t nbFiles, const char* dictFileName, constraint_t target, paramValues_t paramTarget, int cLevelOpt, int cLevelRun)
+static int optimizeForSize(const char* const * const fileNamesTable, const size_t nbFiles, const char* dictFileName, constraint_t target, paramValues_t paramTarget, 
+    int cLevelOpt, int cLevelRun, U32 memoTableLog)
 {
     varInds_t varArray [NUM_PARAMS];
     int ret = 0;
@@ -2217,7 +2202,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_
     paramBase = cParamUnsetMin(paramTarget);
 
     // TODO: if strategy is fixed, only init that row
-    allMT = createMemoTableArray(varArray, varLen);
+    allMT = createMemoTableArray(varArray, varLen, memoTableLog);
 
     if(!allMT) {
         DISPLAY("MemoTable Init Error\n");
@@ -2446,7 +2431,6 @@ static int usage_advanced(void)
     DISPLAY( "\nAdvanced options :\n");
     DISPLAY( " -T#          : set level 1 speed objective \n");
     DISPLAY( " -B#          : cut input into blocks of size # (default : single block) \n");
-    DISPLAY( " -i#          : iteration loops (default : %i) \n", NBLOOPS);
     DISPLAY( " --optimize=  : same as -O with more verbose syntax (see README.md)\n");
     DISPLAY( " -S           : Single run \n");
     DISPLAY( " --zstd       : Single run, parameter selection same as zstdcli \n");
@@ -2497,6 +2481,8 @@ int main(int argc, const char** argv)
     U32 main_pause = 0;
     int cLevelOpt = 0, cLevelRun = 0;
     int seperateFiles = 0;
+    double compressibility = COMPRESSIBILITY_DEFAULT;
+    U32 memoTableLog = PARAM_UNSET;
     constraint_t target = { 0, 0, (U32)-1 }; 
 
     paramValues_t paramTarget = emptyParams();
@@ -2520,7 +2506,7 @@ int main(int argc, const char** argv)
                 PARSE_SUB_ARGS("compressionMemory=" , "cMem=", target.cMem);
                 PARSE_SUB_ARGS("strict=", "stc=", g_strictness);
                 PARSE_SUB_ARGS("maxTries=", "tries=", g_maxTries);
-                PARSE_SUB_ARGS("memoLimitLog=", "memLog=", g_memoTableLog);
+                PARSE_SUB_ARGS("memoLimitLog=", "memLog=", memoTableLog);
                 if (longCommandWArg(&argument, "level=") || longCommandWArg(&argument, "lvl=")) { cLevelOpt = readU32FromChar(&argument); g_optmode = 1; if (argument[0]==',') { argument++; continue; } else break; }
                 if (longCommandWArg(&argument, "speedForRatio=") || longCommandWArg(&argument, "speedRatio=")) { g_ratioMultiplier = readDoubleFromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; }
 
@@ -2600,18 +2586,12 @@ int main(int argc, const char** argv)
 
                     /* Pause at the end (hidden option) */
                 case 'p': main_pause = 1; argument++; break;
-                    /* Modify Nb Iterations */
-
-                case 'i':
-                    argument++;
-                    g_nbIterations = readU32FromChar(&argument);
-                    break;
 
                     /* Sample compressibility (when no file provided) */
                 case 'P':
                     argument++;
                     {   U32 const proba32 = readU32FromChar(&argument);
-                        g_compressibility = (double)proba32 / 100.;
+                        compressibility = (double)proba32 / 100.;
                     }
                     break;
 
@@ -2730,13 +2710,13 @@ int main(int argc, const char** argv)
             DISPLAY("Optimizer Expects File\n");
             return 1;
         } else {
-            result = benchSample();
+            result = benchSample(compressibility);
         }
     } else {
         if(seperateFiles) {
             for(i = 0; i < argc - filenamesStart; i++) {
                 if (g_optimizer) {
-                    result = optimizeForSize(argv+filenamesStart + i, 1, dictFileName, target, paramTarget, cLevelOpt, cLevelRun);
+                    result = optimizeForSize(argv+filenamesStart + i, 1, dictFileName, target, paramTarget, cLevelOpt, cLevelRun, memoTableLog);
                     if(result) { DISPLAY("Error on File %d", i); return result; }
                 } else {
                     result = benchFiles(argv+filenamesStart + i, 1, dictFileName, cLevelRun);
@@ -2745,7 +2725,7 @@ int main(int argc, const char** argv)
             }
         } else {
             if (g_optimizer) {
-                result = optimizeForSize(argv+filenamesStart, argc-filenamesStart, dictFileName, target, paramTarget, cLevelOpt, cLevelRun);
+                result = optimizeForSize(argv+filenamesStart, argc-filenamesStart, dictFileName, target, paramTarget, cLevelOpt, cLevelRun, memoTableLog);
             } else {
                 result = benchFiles(argv+filenamesStart, argc-filenamesStart, dictFileName, cLevelRun);
             }

From 3f8b10baa1b5e70530ec5d893ebaaf8cc9962d04 Mon Sep 17 00:00:00 2001
From: George Lu 
Date: Wed, 15 Aug 2018 14:27:07 -0700
Subject: [PATCH 176/372] consts

---
 tests/paramgrill.c | 26 +++++++++++++++-----------
 1 file changed, 15 insertions(+), 11 deletions(-)

diff --git a/tests/paramgrill.c b/tests/paramgrill.c
index e5e1e2a36..1025e4323 100644
--- a/tests/paramgrill.c
+++ b/tests/paramgrill.c
@@ -1263,7 +1263,6 @@ static void BMK_printWinnerOpt(FILE* f, const U32 cLevel, const BMK_result_t res
         }
     }  
 
-    //prints out tradeoff table if using lvloptimize
     if(g_optmode && g_optimizer && (DEBUG || g_displayLevel == 3)) {
         winnerInfo_t w;
         winner_ll_node* n;
@@ -1574,9 +1573,9 @@ static void freeMemoTableArray(memoTable_t* const mtAll) {
 
 /* inits memotables for all (including mallocs), all strategies */
 /* takes unsanitized varyParams */
-static memoTable_t* createMemoTableArray(const varInds_t* const varyParams, const size_t varyLen, U32 memoTableLog) {
+static memoTable_t* createMemoTableArray(const paramValues_t p, const varInds_t* const varyParams, const size_t varyLen, const U32 memoTableLog) {
     memoTable_t* mtAll = (memoTable_t*)calloc(sizeof(memoTable_t),(ZSTD_btultra + 1));
-    int i;
+    ZSTD_strategy i, stratMin = ZSTD_fast, stratMax = ZSTD_btultra;
     
     if(mtAll == NULL) {
         return NULL;
@@ -1596,8 +1595,14 @@ static memoTable_t* createMemoTableArray(const varInds_t* const varyParams, cons
         return mtAll;
     }
 
-    /* hash table if normal table is too big */
-    for(i = 1; i <= (int)ZSTD_btultra; i++) {
+    
+    if(p.vals[strt_ind] != PARAM_UNSET) {
+        stratMin = p.vals[strt_ind];
+        stratMax = p.vals[strt_ind];
+    }
+
+
+    for(i = stratMin; i <= stratMax; i++) {
         size_t mtl = memoTableLen(mtAll[i].varArray, mtAll[i].varLen);
         mtAll[i].tableType = directMap;
 
@@ -1680,11 +1685,11 @@ static void randomConstrainedParams(paramValues_t* pc, const memoTable_t* memoTa
         int i;
         for(i = 0; i < NUM_PARAMS; i++) {
             varInds_t v = mt.varArray[i];
-            if(v == strt_ind) continue; //skip, already specified
+            if(v == strt_ind) continue; 
             pc->vals[v] = rangeMap(v, FUZ_rand(&g_rand) % rangetable[v]);
         }
 
-        if(!(memoTableGet(memoTableArray, *pc))) break; //only pick unpicked params. 
+        if(!(memoTableGet(memoTableArray, *pc))) break; /* only pick unpicked params. */
     }
 }
 
@@ -2105,7 +2110,7 @@ static winnerInfo_t optimizeFixedStrategy(
 
     for(i = 0; i < tries; i++) {
         DEBUGOUTPUT("Restart\n"); 
-        do { randomConstrainedParams(&init, memoTableArray, strat); } while(redundantParams(init, target, buf.maxBlockSize)); //only non-redundant params  
+        do { randomConstrainedParams(&init, memoTableArray, strat); } while(redundantParams(init, target, buf.maxBlockSize));
         candidateInfo = climbOnce(target, memoTableArray, buf, ctx, init);
         if(compareResultLT(winnerInfo.result, candidateInfo.result, target, buf.srcSize)) {
             winnerInfo = candidateInfo;
@@ -2168,7 +2173,7 @@ static int g_maxTries = 5;
 #define TRY_DECAY 1
 
 static int optimizeForSize(const char* const * const fileNamesTable, const size_t nbFiles, const char* dictFileName, constraint_t target, paramValues_t paramTarget, 
-    int cLevelOpt, int cLevelRun, U32 memoTableLog)
+    const int cLevelOpt, const int cLevelRun, const U32 memoTableLog)
 {
     varInds_t varArray [NUM_PARAMS];
     int ret = 0;
@@ -2201,8 +2206,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_
     optimizerAdjustInput(¶mTarget, buf.maxBlockSize);
     paramBase = cParamUnsetMin(paramTarget);
 
-    // TODO: if strategy is fixed, only init that row
-    allMT = createMemoTableArray(varArray, varLen, memoTableLog);
+    allMT = createMemoTableArray(paramTarget, varArray, varLen, memoTableLog);
 
     if(!allMT) {
         DISPLAY("MemoTable Init Error\n");

From 8a296d3e1f4f8a5177b084098170b122b7e33c26 Mon Sep 17 00:00:00 2001
From: George Lu 
Date: Wed, 15 Aug 2018 14:57:10 -0700
Subject: [PATCH 177/372] Move Stuff around

Group similar functions together, remove outdated comments
---
 tests/paramgrill.c | 1443 ++++++++++++++++++++++----------------------
 1 file changed, 734 insertions(+), 709 deletions(-)

diff --git a/tests/paramgrill.c b/tests/paramgrill.c
index 1025e4323..7ebcdeec0 100644
--- a/tests/paramgrill.c
+++ b/tests/paramgrill.c
@@ -65,8 +65,6 @@ static const int g_maxNbVariations = 64;
 #define CUSTOM_LEVEL 99
 #define BASE_CLEVEL 1
 
-#undef ZSTD_WINDOWLOG_MAX
-#define ZSTD_WINDOWLOG_MAX 27 //no long range stuff for now. 
 #define FADT_MIN 0
 #define FADT_MAX ((U32)-1)
 
@@ -242,7 +240,7 @@ static U32 g_noSeed = 0;
 /* For optimizer */
 static paramValues_t g_params; /* Initialized at the beginning of main w/ emptyParams() function */
 static double g_ratioMultiplier = 5.;
-static U32 g_strictness = PARAM_UNSET; /* range 0 - 99, measure of how strict  */
+static U32 g_strictness = PARAM_UNSET; /* range 1 - 100, measure of how strict  */
 static BMK_result_t g_lvltarget;
 
 typedef enum {
@@ -287,9 +285,23 @@ static winner_ll_node* g_winners; /* linked list sorted ascending by cSize & cSp
  */
 
 /*-*******************************************************
-*  Private functions
+*  General Util Functions 
 *********************************************************/
 
+/* nullified useless params, to ensure count stats */
+/* cleans up params for memoizing / display */
+static paramValues_t sanitizeParams(paramValues_t params)
+{
+    if (params.vals[strt_ind] == ZSTD_fast)
+        params.vals[clog_ind] = 0, params.vals[slog_ind] = 0;
+    if (params.vals[strt_ind] == ZSTD_dfast)
+        params.vals[slog_ind] = 0;
+    if (params.vals[strt_ind] != ZSTD_btopt && params.vals[strt_ind] != ZSTD_btultra && params.vals[strt_ind] != ZSTD_fast)
+        params.vals[tlen_ind] = 0;
+
+    return params;
+}
+
 static ZSTD_compressionParameters pvalsToCParams(paramValues_t p) {
     ZSTD_compressionParameters c;
     memset(&c, 0, sizeof(ZSTD_compressionParameters));
@@ -335,9 +347,6 @@ static paramValues_t adjustParams(paramValues_t p, const size_t maxBlockSize, co
     return p;
 }
 
-/* accuracy in seconds only, span can be multiple years */
-static U32 BMK_timeSpan(const UTIL_time_t tStart) { return (U32)(UTIL_clockSpanMicro(tStart) / 1000000ULL); }
-
 static size_t BMK_findMaxMem(U64 requiredMem)
 {
     size_t const step = 64 MB;
@@ -356,6 +365,8 @@ static size_t BMK_findMaxMem(U64 requiredMem)
     return (size_t) requiredMem;
 }
 
+/* accuracy in seconds only, span can be multiple years */
+static U32 BMK_timeSpan(const UTIL_time_t tStart) { return (U32)(UTIL_clockSpanMicro(tStart) / 1000000ULL); }
 
 static U32 FUZ_rotl32(U32 x, U32 r)
 {
@@ -374,42 +385,6 @@ U32 FUZ_rand(U32* src)
     return rand32 >> 5;
 }
 
-/** longCommandWArg() :
- *  check if *stringPtr is the same as longCommand.
- *  If yes, @return 1 and advances *stringPtr to the position which immediately follows longCommand.
- * @return 0 and doesn't modify *stringPtr otherwise.
- * from zstdcli.c
- */
-static unsigned longCommandWArg(const char** stringPtr, const char* longCommand)
-{
-    size_t const comSize = strlen(longCommand);
-    int const result = !strncmp(*stringPtr, longCommand, comSize);
-    if (result) *stringPtr += comSize;
-    return result;
-}
-
-static U64 g_clockGranularity = 100000000ULL;
-
-static void findClockGranularity(void) {
-    UTIL_time_t clockStart = UTIL_getTime();
-    U64 el1 = 0, el2 = 0;
-    int i = 0;
-    do {
-        el1 = el2;
-        el2 = UTIL_clockSpanNano(clockStart);
-        if(el1 < el2) {
-            U64 iv = el2 - el1;
-            if(g_clockGranularity > iv) {
-                g_clockGranularity = iv;
-                i = 0;
-            } else {
-                i++;
-            }
-        }
-    } while(i < 10);
-    DEBUGOUTPUT("Granularity: %llu\n", (unsigned long long)g_clockGranularity);
-}
-
 /* allows zeros */
 #define CLAMPCHECK(val,min,max) {                     \
     if (((val)<(min)) | ((val)>(max))) {              \
@@ -435,38 +410,95 @@ static paramValues_t cParamUnsetMin(paramValues_t paramTarget) {
     return paramTarget;
 }
 
-static void BMK_translateAdvancedParams(FILE* f, const paramValues_t params) {
-    varInds_t v;
-    int first = 1;
-    fprintf(f,"--zstd=");
-    for(v = 0; v < NUM_PARAMS; v++) {
-        if(g_silenceParams[v]) { continue; }
-        if(!first) { fprintf(f, ","); }
-        fprintf(f,"%s=", g_paramNames[v]);
-        
-        if(v == strt_ind) { fprintf(f,"%u", params.vals[v]); }
-        else { displayParamVal(f, v, params.vals[v], 0); }
-        first = 0;
+static paramValues_t emptyParams(void) {
+    U32 i;
+    paramValues_t p;
+    for(i = 0; i < NUM_PARAMS; i++) {
+        p.vals[i] = PARAM_UNSET;
     }
-    fprintf(f, "\n");
+    return p;
 }
 
-static void BMK_displayOneResult(FILE* f, winnerInfo_t res, const size_t srcSize) {
-            varInds_t v;
-            int first = 1;
-            res.params = cParamUnsetMin(res.params);
-            fprintf(f,"    {");
-            for(v = 0; v < NUM_PARAMS; v++) {
-                if(g_silenceParams[v]) { continue; }
-                if(!first) { fprintf(f, ","); }
-                displayParamVal(f, v, res.params.vals[v], 3);
-                first = 0;
+static winnerInfo_t initWinnerInfo(const paramValues_t p) {
+    winnerInfo_t w1;
+    w1.result.cSpeed = 0.;
+    w1.result.dSpeed = 0.;
+    w1.result.cMem = (size_t)-1;
+    w1.result.cSize = (size_t)-1;
+    w1.params = p;
+    return w1;
+}
+
+static paramValues_t overwriteParams(paramValues_t base, const paramValues_t mask) {
+    U32 i;
+    for(i = 0; i < NUM_PARAMS; i++) {
+        if(mask.vals[i] != PARAM_UNSET) {
+            base.vals[i] = mask.vals[i];
+        }
+    }
+    return base;
+}
+
+/* amt will probably always be \pm 1? */
+/* slight change from old paramVariation, targetLength can only take on powers of 2 now (999 ~= 1024?) */
+/* take max/min bounds into account as well? */
+static void paramVaryOnce(const varInds_t paramIndex, const int amt, paramValues_t* ptr) {
+    ptr->vals[paramIndex] = rangeMap(paramIndex, invRangeMap(paramIndex, ptr->vals[paramIndex]) + amt);
+}
+
+/* varies ptr by nbChanges respecting varyParams*/
+static void paramVariation(paramValues_t* ptr, memoTable_t* mtAll, const U32 nbChanges)
+{
+    paramValues_t p;
+    U32 validated = 0;
+    while (!validated) {
+        U32 i;
+        p = *ptr;
+        for (i = 0 ; i < nbChanges ; i++) {
+            const U32 changeID = (U32)FUZ_rand(&g_rand) % (mtAll[p.vals[strt_ind]].varLen << 1);
+            paramVaryOnce(mtAll[p.vals[strt_ind]].varArray[changeID >> 1], ((changeID & 1) << 1) - 1, &p);
+        }
+        validated = paramValid(p);
+    }
+    *ptr = p;
+}
+
+/* Completely random parameter selection */
+static paramValues_t randomParams(void)
+{
+    varInds_t v; paramValues_t p;
+    for(v = 0; v <= NUM_PARAMS; v++) {
+        p.vals[v] = rangeMap(v, FUZ_rand(&g_rand) % rangetable[v]);
+    }
+    return p;
+}
+
+static U64 g_clockGranularity = 100000000ULL;
+
+static void findClockGranularity(void) {
+    UTIL_time_t clockStart = UTIL_getTime();
+    U64 el1 = 0, el2 = 0;
+    int i = 0;
+    do {
+        el1 = el2;
+        el2 = UTIL_clockSpanNano(clockStart);
+        if(el1 < el2) {
+            U64 iv = el2 - el1;
+            if(g_clockGranularity > iv) {
+                g_clockGranularity = iv;
+                i = 0;
+            } else {
+                i++;
             }
-
-            fprintf(f, " },     /* R:%5.3f at %5.1f MB/s - %5.1f MB/s */\n",
-            (double)srcSize / res.result.cSize, (double)res.result.cSpeed / (1 MB), (double)res.result.dSpeed / (1 MB));
+        }
+    } while(i < 10);
+    DEBUGOUTPUT("Granularity: %llu\n", (unsigned long long)g_clockGranularity);
 }
 
+/*-************************************
+*  Optimizer Util Functions
+**************************************/
+
 /* checks results are feasible */
 static int feasible(const BMK_result_t results, const constraint_t target) {
     return (results.cSpeed >= target.cSpeed) && (results.dSpeed >= target.dSpeed) && (results.cMem <= target.cMem) && (!g_optmode || results.cSize <= g_lvltarget.cSize);
@@ -523,52 +555,324 @@ static constraint_t relaxTarget(constraint_t target) {
     return target;
 }
 
-/*-*******************************************************
-*  Bench functions
-*********************************************************/
-
-static paramValues_t emptyParams(void) {
-    U32 i;
-    paramValues_t p;
-    for(i = 0; i < NUM_PARAMS; i++) {
-        p.vals[i] = PARAM_UNSET;
+static void optimizerAdjustInput(paramValues_t* pc, const size_t maxBlockSize) {
+    varInds_t v;
+    for(v = 0; v < NUM_PARAMS; v++) {
+        if(pc->vals[v] != PARAM_UNSET) {
+            U32 newval = MIN(MAX(pc->vals[v], mintable[v]), maxtable[v]);
+            if(newval != pc->vals[v]) {
+                pc->vals[v] = newval;
+                DISPLAY("Warning: parameter %s not in valid range, adjusting to ", g_paramNames[v]); displayParamVal(stderr, v, newval, 0); DISPLAY("\n");
+            }
+        }
+    }
+
+    if(pc->vals[wlog_ind] != PARAM_UNSET) {
+
+        U32 sshb = maxBlockSize > 1 ? ZSTD_highbit32((U32)(maxBlockSize-1)) + 1 : 1;
+        /* edge case of highBit not working for 0 */
+
+        if(maxBlockSize < (1ULL << 31) && sshb + 1 < pc->vals[wlog_ind]) {
+            U32 adjust = MAX(mintable[wlog_ind], sshb);
+            if(adjust != pc->vals[wlog_ind]) {
+                pc->vals[wlog_ind] = adjust;
+                DISPLAY("Warning: windowLog larger than src/block size, adjusted to %u\n", pc->vals[wlog_ind]);
+            }
+        }
+    }
+
+    if(pc->vals[wlog_ind] != PARAM_UNSET && pc->vals[clog_ind] != PARAM_UNSET) {
+        U32 maxclog;
+        if(pc->vals[strt_ind] == PARAM_UNSET || pc->vals[strt_ind] >= (U32)ZSTD_btlazy2) {
+            maxclog = pc->vals[wlog_ind] + 1;
+        } else {
+            maxclog = pc->vals[wlog_ind];
+        }
+
+        if(pc->vals[clog_ind] > maxclog) {
+            pc->vals[clog_ind] = maxclog;
+            DISPLAY("Warning: chainlog too much larger than windowLog size, adjusted to %u\n", pc->vals[clog_ind]);
+        }
+    }
+
+    if(pc->vals[wlog_ind] != PARAM_UNSET && pc->vals[hlog_ind] != PARAM_UNSET) {
+        if(pc->vals[wlog_ind] + 1 < pc->vals[hlog_ind]) {
+            pc->vals[hlog_ind] = pc->vals[wlog_ind] + 1;
+            DISPLAY("Warning: hashlog too much larger than windowLog size, adjusted to %u\n", pc->vals[hlog_ind]);
+        }
+    }
+    
+    if(pc->vals[slog_ind] != PARAM_UNSET && pc->vals[clog_ind] != PARAM_UNSET) {
+        if(pc->vals[slog_ind] > pc->vals[clog_ind]) {
+            pc->vals[clog_ind] = pc->vals[slog_ind];
+            DISPLAY("Warning: searchLog larger than chainLog, adjusted to %u\n", pc->vals[slog_ind]);
+        }
     }
-    return p;
 }
 
-static winnerInfo_t initWinnerInfo(const paramValues_t p) {
-    winnerInfo_t w1;
-    w1.result.cSpeed = 0.;
-    w1.result.dSpeed = 0.;
-    w1.result.cMem = (size_t)-1;
-    w1.result.cSize = (size_t)-1;
-    w1.params = p;
-    return w1;
+/* what about low something like clog vs hlog in lvl 1?  */
+static int redundantParams(const paramValues_t paramValues, const constraint_t target, const size_t maxBlockSize) {
+    return 
+       (ZSTD_estimateCStreamSize_usingCParams(pvalsToCParams(paramValues)) > (size_t)target.cMem) /* Uses too much memory */
+    || ((1ULL << (paramValues.vals[wlog_ind] - 1)) >= maxBlockSize && paramValues.vals[wlog_ind] != mintable[wlog_ind]) /* wlog too much bigger than src size */
+    || (paramValues.vals[clog_ind] > (paramValues.vals[wlog_ind] + (paramValues.vals[strt_ind] > ZSTD_btlazy2))) /* chainLog larger than windowLog*/
+    || (paramValues.vals[slog_ind] > paramValues.vals[clog_ind]) /* searchLog larger than chainLog */
+    || (paramValues.vals[hlog_ind] > paramValues.vals[wlog_ind] + 1); /* hashLog larger than windowLog + 1 */
+    
 }
 
-typedef struct {
-    void* srcBuffer;
-    size_t srcSize;
-    const void** srcPtrs;
-    size_t* srcSizes;
-    void** dstPtrs;
-    size_t* dstCapacities;
-    size_t* dstSizes;
-    void** resPtrs;
-    size_t* resSizes;
-    size_t nbBlocks;
-    size_t maxBlockSize;
-} buffers_t;
+/*-************************************
+*  Display Functions 
+**************************************/
+
+static void BMK_translateAdvancedParams(FILE* f, const paramValues_t params) {
+    varInds_t v;
+    int first = 1;
+    fprintf(f,"--zstd=");
+    for(v = 0; v < NUM_PARAMS; v++) {
+        if(g_silenceParams[v]) { continue; }
+        if(!first) { fprintf(f, ","); }
+        fprintf(f,"%s=", g_paramNames[v]);
+        
+        if(v == strt_ind) { fprintf(f,"%u", params.vals[v]); }
+        else { displayParamVal(f, v, params.vals[v], 0); }
+        first = 0;
+    }
+    fprintf(f, "\n");
+}
+
+static void BMK_displayOneResult(FILE* f, winnerInfo_t res, const size_t srcSize) {
+            varInds_t v;
+            int first = 1;
+            res.params = cParamUnsetMin(res.params);
+            fprintf(f,"    {");
+            for(v = 0; v < NUM_PARAMS; v++) {
+                if(g_silenceParams[v]) { continue; }
+                if(!first) { fprintf(f, ","); }
+                displayParamVal(f, v, res.params.vals[v], 3);
+                first = 0;
+            }
+
+            fprintf(f, " },     /* R:%5.3f at %5.1f MB/s - %5.1f MB/s */\n",
+            (double)srcSize / res.result.cSize, (double)res.result.cSpeed / (1 MB), (double)res.result.dSpeed / (1 MB));
+}
+
+/* Writes to f the results of a parameter benchmark */
+/* when used with --optimize, will only print results better than previously discovered */
+static void BMK_printWinner(FILE* f, const U32 cLevel, const BMK_result_t result, const paramValues_t params, const size_t srcSize)
+{
+    char lvlstr[15] = "Custom Level";
+    winnerInfo_t w;
+    w.params = params;
+    w.result = result;
+
+    fprintf(f, "\r%79s\r", "");
+
+    if(cLevel != CUSTOM_LEVEL) {
+        snprintf(lvlstr, 15, "  Level %2u  ", cLevel);
+    }
+
+    if(TIMED) { 
+        const U64 time = UTIL_clockSpanNano(g_time);
+        const U64 minutes = time / (60ULL * TIMELOOP_NANOSEC);
+        fprintf(f, "%1lu:%2lu:%05.2f - ", (unsigned long) minutes / 60,(unsigned long) minutes % 60, (double)(time - minutes * TIMELOOP_NANOSEC * 60ULL)/TIMELOOP_NANOSEC); 
+    }
+
+    fprintf(f, "/* %s */   ", lvlstr);
+    BMK_displayOneResult(f, w, srcSize);
+}
+
+/* comparison function: */
+/* strictly better, strictly worse, equal, speed-side adv, size-side adv */
+//Maybe use compress_only for benchmark first run?
+#define WORSE_RESULT 0
+#define BETTER_RESULT 1
+#define ERROR_RESULT 2
+
+#define SPEED_RESULT 4
+#define SIZE_RESULT 5
+/* maybe have epsilon-eq to limit table size? */
+static int speedSizeCompare(const BMK_result_t r1, const BMK_result_t r2) {
+    if(r1.cSpeed < r2.cSpeed) {
+        if(r1.cSize >= r2.cSize) {
+            return BETTER_RESULT;
+        }
+        return SPEED_RESULT; /* r2 is smaller but not faster. */
+    } else {
+        if(r1.cSize <= r2.cSize) {
+            return WORSE_RESULT;
+        }
+        return SIZE_RESULT; /* r2 is faster but not smaller */
+    }
+}
+
+/* 0 for insertion, 1 for no insert */
+/* maintain invariant speedSizeCompare(n, n->next) = SPEED_RESULT */
+static int insertWinner(const winnerInfo_t w, const constraint_t targetConstraints) {
+    BMK_result_t r = w.result;
+    winner_ll_node* cur_node = g_winners;
+    /* first node to insert */
+    if(!feasible(r, targetConstraints)) {
+        return 1;
+    }
+
+    if(g_winners == NULL) {
+        winner_ll_node* first_node = malloc(sizeof(winner_ll_node));
+        if(first_node == NULL) {
+            return 1;
+        }
+        first_node->next = NULL;
+        first_node->res = w;
+        g_winners = first_node;
+        return 0;
+    }
+
+    while(cur_node->next != NULL) {
+        switch(speedSizeCompare(cur_node->res.result, r)) {
+            case WORSE_RESULT:
+            {
+                return 1; /* never insert if better */
+            }
+            case BETTER_RESULT:
+            {
+                winner_ll_node* tmp;
+                cur_node->res = cur_node->next->res;
+                tmp = cur_node->next;
+                cur_node->next = cur_node->next->next;
+                free(tmp);
+                break; 
+            }
+            case SIZE_RESULT:
+            {
+                cur_node = cur_node->next;
+                break;
+            }
+            case SPEED_RESULT: /* insert after first size result, then return */
+            {
+                winner_ll_node* newnode = malloc(sizeof(winner_ll_node));
+                if(newnode == NULL) {
+                    return 1;
+                }
+                newnode->res = cur_node->res;
+                cur_node->res = w;
+                newnode->next = cur_node->next;
+                cur_node->next = newnode;
+                return 0;
+            }
+        } 
+
+    }
+
+    assert(cur_node->next == NULL);
+    switch(speedSizeCompare(cur_node->res.result, r)) {
+        case WORSE_RESULT:
+        {
+            return 1; /* never insert if better */
+        }
+        case BETTER_RESULT:
+        {
+            cur_node->res = w;
+            return 0;
+        }
+        case SIZE_RESULT:
+        {
+            winner_ll_node* newnode = malloc(sizeof(winner_ll_node));
+            if(newnode == NULL) {
+                return 1;
+            }
+            newnode->res = w;
+            newnode->next = NULL;
+            cur_node->next = newnode;
+            return 0;
+        }
+        case SPEED_RESULT: /* insert before first size result, then return */
+        {
+            winner_ll_node* newnode = malloc(sizeof(winner_ll_node));
+            if(newnode == NULL) {
+                return 1;
+            }
+            newnode->res = cur_node->res;
+            cur_node->res = w;
+            newnode->next = cur_node->next;
+            cur_node->next = newnode;
+            return 0;
+        }
+        default: 
+            return 1;
+    } 
+}
+
+static void BMK_printWinnerOpt(FILE* f, const U32 cLevel, const BMK_result_t result, const paramValues_t params, const constraint_t targetConstraints, const size_t srcSize)
+{
+    /* global winner used for constraints */
+                                    /* cSize, cSpeed, dSpeed, cMem */
+    static winnerInfo_t g_winner = { { (size_t)-1LL, 0, 0, (size_t)-1LL }, { { PARAM_UNSET, PARAM_UNSET, PARAM_UNSET, PARAM_UNSET, PARAM_UNSET, PARAM_UNSET, PARAM_UNSET, PARAM_UNSET } } };
+    if(DEBUG || compareResultLT(g_winner.result, result, targetConstraints, srcSize) || g_displayLevel >= 4) {
+        if(DEBUG && compareResultLT(g_winner.result, result, targetConstraints, srcSize)) {
+            DISPLAY("New Winner: \n");
+        }
+
+        if(g_displayLevel >= 2) { BMK_printWinner(f, cLevel, result, params, srcSize); }
+
+        if(compareResultLT(g_winner.result, result, targetConstraints, srcSize)) {
+            if(g_displayLevel >= 1) { BMK_translateAdvancedParams(f, params); }
+            g_winner.result = result;
+            g_winner.params = params;
+        }
+    }  
+
+    if(g_optmode && g_optimizer && (DEBUG || g_displayLevel == 3)) {
+        winnerInfo_t w;
+        winner_ll_node* n;
+        w.result = result;
+        w.params = params;
+        insertWinner(w, targetConstraints);
+
+        if(!DEBUG) { fprintf(f, "\033c"); }
+        fprintf(f, "\n"); 
+        
+        /* the table */
+        fprintf(f, "================================\n");
+        for(n = g_winners; n != NULL; n = n->next) {
+            BMK_displayOneResult(f, n->res, srcSize);
+        }
+        fprintf(f, "================================\n");
+        fprintf(f, "Level Bounds: R: > %.3f AND C: < %.1f MB/s \n\n",
+            (double)srcSize / g_lvltarget.cSize, (double)g_lvltarget.cSpeed / (1 MB));
+
+
+        fprintf(f, "Overall Winner: \n");
+        BMK_displayOneResult(f, g_winner, srcSize);
+        BMK_translateAdvancedParams(f, g_winner.params);
+
+        fprintf(f, "Latest BMK: \n");\
+        BMK_displayOneResult(f, w, srcSize);
+    }
+}
+
+static void BMK_printWinners2(FILE* f, const winnerInfo_t* winners, const size_t srcSize)
+{
+    int cLevel;
+
+    fprintf(f, "\n /* Proposed configurations : */ \n");
+    fprintf(f, "    /* W,  C,  H,  S,  L,  T, strat */ \n");
+
+    for (cLevel=0; cLevel <= NB_LEVELS_TRACKED; cLevel++)
+        BMK_printWinner(f, cLevel, winners[cLevel].result, winners[cLevel].params, srcSize);
+}
+
+
+static void BMK_printWinners(FILE* f, const winnerInfo_t* winners, const size_t srcSize)
+{
+    fseek(f, 0, SEEK_SET);
+    BMK_printWinners2(f, winners, srcSize);
+    fflush(f);
+    BMK_printWinners2(stdout, winners, srcSize);
+}
 
-typedef struct {
-    size_t dictSize;
-    void* dictBuffer;
-    ZSTD_CCtx* cctx;
-    ZSTD_DCtx* dctx;
-} contexts_t;
 
 /*-*******************************************************
-*  From bench.c
+*  Functions to Benchmark
 *********************************************************/
 
 typedef struct {
@@ -665,75 +969,30 @@ static size_t local_defaultDecompress(
 
 }
 
-/*-*******************************************************
-*  From bench.c End
-*********************************************************/
+/*-************************************
+*  Data Initialization Functions 
+**************************************/
 
-static void optimizerAdjustInput(paramValues_t* pc, const size_t maxBlockSize) {
-    varInds_t v;
-    for(v = 0; v < NUM_PARAMS; v++) {
-        if(pc->vals[v] != PARAM_UNSET) {
-            U32 newval = MIN(MAX(pc->vals[v], mintable[v]), maxtable[v]);
-            if(newval != pc->vals[v]) {
-                pc->vals[v] = newval;
-                DISPLAY("Warning: parameter %s not in valid range, adjusting to ", g_paramNames[v]); displayParamVal(stderr, v, newval, 0); DISPLAY("\n");
-            }
-        }
-    }
+typedef struct {
+    void* srcBuffer;
+    size_t srcSize;
+    const void** srcPtrs;
+    size_t* srcSizes;
+    void** dstPtrs;
+    size_t* dstCapacities;
+    size_t* dstSizes;
+    void** resPtrs;
+    size_t* resSizes;
+    size_t nbBlocks;
+    size_t maxBlockSize;
+} buffers_t;
 
-    if(pc->vals[wlog_ind] != PARAM_UNSET) {
-
-        U32 sshb = maxBlockSize > 1 ? ZSTD_highbit32((U32)(maxBlockSize-1)) + 1 : 1;
-        /* edge case of highBit not working for 0 */
-
-        if(maxBlockSize < (1ULL << 31) && sshb + 1 < pc->vals[wlog_ind]) {
-            U32 adjust = MAX(mintable[wlog_ind], sshb);
-            if(adjust != pc->vals[wlog_ind]) {
-                pc->vals[wlog_ind] = adjust;
-                DISPLAY("Warning: windowLog larger than src/block size, adjusted to %u\n", pc->vals[wlog_ind]);
-            }
-        }
-    }
-
-    if(pc->vals[wlog_ind] != PARAM_UNSET && pc->vals[clog_ind] != PARAM_UNSET) {
-        U32 maxclog;
-        if(pc->vals[strt_ind] == PARAM_UNSET || pc->vals[strt_ind] >= (U32)ZSTD_btlazy2) {
-            maxclog = pc->vals[wlog_ind] + 1;
-        } else {
-            maxclog = pc->vals[wlog_ind];
-        }
-
-        if(pc->vals[clog_ind] > maxclog) {
-            pc->vals[clog_ind] = maxclog;
-            DISPLAY("Warning: chainlog too much larger than windowLog size, adjusted to %u\n", pc->vals[clog_ind]);
-        }
-    }
-
-    if(pc->vals[wlog_ind] != PARAM_UNSET && pc->vals[hlog_ind] != PARAM_UNSET) {
-        if(pc->vals[wlog_ind] + 1 < pc->vals[hlog_ind]) {
-            pc->vals[hlog_ind] = pc->vals[wlog_ind] + 1;
-            DISPLAY("Warning: hashlog too much larger than windowLog size, adjusted to %u\n", pc->vals[hlog_ind]);
-        }
-    }
-    
-    if(pc->vals[slog_ind] != PARAM_UNSET && pc->vals[clog_ind] != PARAM_UNSET) {
-        if(pc->vals[slog_ind] > pc->vals[clog_ind]) {
-            pc->vals[clog_ind] = pc->vals[slog_ind];
-            DISPLAY("Warning: searchLog larger than chainLog, adjusted to %u\n", pc->vals[slog_ind]);
-        }
-    }
-}
-
-/* what about low something like clog vs hlog in lvl 1?  */
-static int redundantParams(const paramValues_t paramValues, const constraint_t target, const size_t maxBlockSize) {
-    return 
-       (ZSTD_estimateCStreamSize_usingCParams(pvalsToCParams(paramValues)) > (size_t)target.cMem) /* Uses too much memory */
-    || ((1ULL << (paramValues.vals[wlog_ind] - 1)) >= maxBlockSize && paramValues.vals[wlog_ind] != mintable[wlog_ind]) /* wlog too much bigger than src size */
-    || (paramValues.vals[clog_ind] > (paramValues.vals[wlog_ind] + (paramValues.vals[strt_ind] > ZSTD_btlazy2))) /* chainLog larger than windowLog*/
-    || (paramValues.vals[slog_ind] > paramValues.vals[clog_ind]) /* searchLog larger than chainLog */
-    || (paramValues.vals[hlog_ind] > paramValues.vals[wlog_ind] + 1); /* hashLog larger than windowLog + 1 */
-    
-}
+typedef struct {
+    size_t dictSize;
+    void* dictBuffer;
+    ZSTD_CCtx* cctx;
+    ZSTD_DCtx* dctx;
+} contexts_t;
 
 static void freeNonSrcBuffers(const buffers_t b) {
     free(b.srcPtrs);
@@ -952,6 +1211,175 @@ static int createContexts(contexts_t* ctx, const char* dictFileName) {
     return 0;
 }
 
+/*-************************************
+*  Optimizer Memoization Functions 
+**************************************/
+
+/* return: new length */
+/* keep old array, will need if iter over strategy. */
+/* prunes useless params */
+static size_t sanitizeVarArray(varInds_t* varNew, const size_t varLength, const varInds_t* varArray, const ZSTD_strategy strat) {
+    size_t i, j = 0;
+    for(i = 0; i < varLength; i++) {
+        if( !((varArray[i] == clog_ind && strat == ZSTD_fast)
+            || (varArray[i] == slog_ind && strat == ZSTD_fast)
+            || (varArray[i] == slog_ind && strat == ZSTD_dfast) 
+            || (varArray[i] == tlen_ind && strat != ZSTD_btopt && strat != ZSTD_btultra && strat != ZSTD_fast))) {
+            varNew[j] = varArray[i];
+            j++;
+        }
+    }
+    return j;
+}
+
+/* res should be NUM_PARAMS size */
+/* constructs varArray from paramValues_t style parameter */
+/* pass in using dict. */
+static size_t variableParams(const paramValues_t paramConstraints, varInds_t* res, const int usingDictionary) {
+    varInds_t i;
+    size_t j = 0;
+    for(i = 0; i < NUM_PARAMS; i++) {
+        if(paramConstraints.vals[i] == PARAM_UNSET) {
+            if(i == fadt_ind && !usingDictionary) continue; /* don't use fadt if no dictionary */
+            res[j] = i; j++;
+        }
+    }
+    return j;
+}
+
+/* length of memo table given free variables */
+static size_t memoTableLen(const varInds_t* varyParams, const size_t varyLen) {
+    size_t arrayLen = 1;
+    size_t i;
+    for(i = 0; i < varyLen; i++) {
+        if(varyParams[i] == strt_ind) continue; /* strategy separated by table */
+        arrayLen *= rangetable[varyParams[i]];
+    }
+    return arrayLen;
+}
+
+/* returns unique index in memotable of compression parameters */
+static unsigned memoTableIndDirect(const paramValues_t* ptr, const varInds_t* varyParams, const size_t varyLen) {
+    size_t i;
+    unsigned ind = 0;
+    for(i = 0; i < varyLen; i++) {
+        varInds_t v = varyParams[i];
+        if(v == strt_ind) continue; /* exclude strategy from memotable */
+        ind *= rangetable[v]; ind += (unsigned)invRangeMap(v, ptr->vals[v]);
+    }
+    return ind;
+}
+
+static size_t memoTableGet(const memoTable_t* memoTableArray, const paramValues_t p) {
+    const memoTable_t mt = memoTableArray[p.vals[strt_ind]];
+    switch(mt.tableType) {
+        case directMap:
+            return mt.table[memoTableIndDirect(&p, mt.varArray, mt.varLen)];
+        case xxhashMap:
+            return mt.table[(XXH64(&p.vals, sizeof(U32) * NUM_PARAMS, 0) >> 3) % mt.tableLen];
+        case noMemo:
+            return 0;
+    }
+    return 0; /* should never happen, stop compiler warnings */
+}
+
+static void memoTableSet(const memoTable_t* memoTableArray, const paramValues_t p, const BYTE value) {
+    const memoTable_t mt = memoTableArray[p.vals[strt_ind]];
+    switch(mt.tableType) {
+        case directMap:
+            mt.table[memoTableIndDirect(&p, mt.varArray, mt.varLen)] = value; break;
+        case xxhashMap:
+            mt.table[(XXH64(&p.vals, sizeof(U32) * NUM_PARAMS, 0) >> 3) % mt.tableLen] = value; break;
+        case noMemo:
+            break;
+    }
+}
+
+/* frees all allocated memotables */
+static void freeMemoTableArray(memoTable_t* const mtAll) {
+    int i;
+    if(mtAll == NULL) { return; }
+    for(i = 1; i <= (int)ZSTD_btultra; i++) {
+        free(mtAll[i].table);
+    }
+    free(mtAll);
+}
+
+/* inits memotables for all (including mallocs), all strategies */
+/* takes unsanitized varyParams */
+static memoTable_t* createMemoTableArray(const paramValues_t p, const varInds_t* const varyParams, const size_t varyLen, const U32 memoTableLog) {
+    memoTable_t* mtAll = (memoTable_t*)calloc(sizeof(memoTable_t),(ZSTD_btultra + 1));
+    ZSTD_strategy i, stratMin = ZSTD_fast, stratMax = ZSTD_btultra;
+    
+    if(mtAll == NULL) {
+        return NULL;
+    }
+
+    for(i = 1; i <= (int)ZSTD_btultra; i++) {
+        mtAll[i].varLen = sanitizeVarArray(mtAll[i].varArray, varyLen, varyParams, i);
+    }
+
+    /* no memoization */
+    if(memoTableLog == 0) {
+        for(i = 1; i <= (int)ZSTD_btultra; i++) {
+            mtAll[i].tableType = noMemo;
+            mtAll[i].table = NULL;
+            mtAll[i].tableLen = 0;
+        }
+        return mtAll;
+    }
+
+    
+    if(p.vals[strt_ind] != PARAM_UNSET) {
+        stratMin = p.vals[strt_ind];
+        stratMax = p.vals[strt_ind];
+    }
+
+
+    for(i = stratMin; i <= stratMax; i++) {
+        size_t mtl = memoTableLen(mtAll[i].varArray, mtAll[i].varLen);
+        mtAll[i].tableType = directMap;
+
+        if(memoTableLog != PARAM_UNSET && mtl > (1ULL << memoTableLog)) { /* use hash table */ /* provide some option to only use hash tables? */
+            mtAll[i].tableType = xxhashMap;
+            mtl = (1ULL << memoTableLog);
+        }
+
+        mtAll[i].table = (BYTE*)calloc(sizeof(BYTE), mtl);
+        mtAll[i].tableLen = mtl;
+
+        if(mtAll[i].table == NULL) {
+            freeMemoTableArray(mtAll);
+            return NULL;
+        }
+    }
+    
+    return mtAll;
+}
+
+/* Sets pc to random unmeasured set of parameters */
+/* specifiy strategy */
+static void randomConstrainedParams(paramValues_t* pc, const memoTable_t* memoTableArray, const ZSTD_strategy st)
+{
+    size_t j;
+    const memoTable_t mt = memoTableArray[st];
+    pc->vals[strt_ind] = st;
+    for(j = 0; j < mt.tableLen; j++) {
+        int i;
+        for(i = 0; i < NUM_PARAMS; i++) {
+            varInds_t v = mt.varArray[i];
+            if(v == strt_ind) continue; 
+            pc->vals[v] = rangeMap(v, FUZ_rand(&g_rand) % rangetable[v]);
+        }
+
+        if(!(memoTableGet(memoTableArray, *pc))) break; /* only pick unpicked params. */
+    }
+}
+
+/*-************************************
+*  Benchmarking Functions
+**************************************/
+
 /* Replicate functionality of benchMemAdvanced, but with pre-split src / dst buffers */
 /* The purpose is so that sufficient information is returned so that a decompression call to benchMemInvertible is possible */
 /* BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, fileSizes, nbFiles, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); */
@@ -1099,220 +1527,115 @@ static int BMK_benchParam(BMK_result_t* resultPtr,
     return res.error;
 }
 
-/* comparison function: */
-/* strictly better, strictly worse, equal, speed-side adv, size-side adv */
-//Maybe use compress_only for benchmark first run?
-#define WORSE_RESULT 0
-#define BETTER_RESULT 1
-#define ERROR_RESULT 2
 
-#define SPEED_RESULT 4
-#define SIZE_RESULT 5
-/* maybe have epsilon-eq to limit table size? */
-static int speedSizeCompare(const BMK_result_t r1, const BMK_result_t r2) {
-    if(r1.cSpeed < r2.cSpeed) {
-        if(r1.cSize >= r2.cSize) {
-            return BETTER_RESULT;
-        }
-        return SPEED_RESULT; /* r2 is smaller but not faster. */
-    } else {
-        if(r1.cSize <= r2.cSize) {
-            return WORSE_RESULT;
-        }
-        return SIZE_RESULT; /* r2 is faster but not smaller */
+#define CBENCHMARK(conditional, resultvar, tmpret, mode, loopmode, sec) {                                       \
+    if(conditional) {                                                                                           \
+        BMK_return_t tmpret = BMK_benchMemInvertible(buf, ctx, BASE_CLEVEL, &cParams, mode, loopmode, sec);     \
+        if(tmpret.error) { DEBUGOUTPUT("Benchmarking failed\n"); return ERROR_RESULT; }                         \
+        if(mode != BMK_decodeOnly)  {                                                                           \
+            resultvar.cSpeed = tmpret.result.cSpeed;                                                            \
+            resultvar.cSize = tmpret.result.cSize;                                                              \
+            resultvar.cMem = tmpret.result.cMem;                                                                \
+        }                                                                                                       \
+        if(mode != BMK_compressOnly) { resultvar.dSpeed = tmpret.result.dSpeed; }                               \
+    }                                                                                                           \
+}
+
+/* Benchmarking which stops when we are sufficiently sure the solution is infeasible / worse than the winner */
+#define VARIANCE 1.2 
+static int allBench(BMK_result_t* resultPtr,
+                const buffers_t buf, const contexts_t ctx,
+                const paramValues_t cParams,
+                const constraint_t target,
+                BMK_result_t* winnerResult, int feas) {
+    BMK_result_t resultMax, benchres;
+    U64 loopDurationC = 0, loopDurationD = 0;
+    double uncertaintyConstantC = 3., uncertaintyConstantD = 3.;
+    double winnerRS;
+    /* initial benchmarking, gives exact ratio and memory, warms up future runs */
+    CBENCHMARK(1, benchres, tmp, BMK_both, BMK_iterMode, 1);
+
+    winnerRS = resultScore(*winnerResult, buf.srcSize, target);
+    DEBUGOUTPUT("WinnerScore: %f\n ", winnerRS);
+
+    *resultPtr = benchres;
+
+    /* calculate uncertainty in compression / decompression runs */
+    if(benchres.cSpeed) {
+        loopDurationC = ((buf.srcSize * TIMELOOP_NANOSEC) / benchres.cSpeed); 
+        uncertaintyConstantC = ((loopDurationC + (double)(2 * g_clockGranularity))/loopDurationC); 
+    }
+
+    if(benchres.dSpeed) {
+        loopDurationD = ((buf.srcSize * TIMELOOP_NANOSEC) / benchres.dSpeed); 
+        uncertaintyConstantD = ((loopDurationD + (double)(2 * g_clockGranularity))/loopDurationD);  
+    }
+
+    /* anything with worse ratio in feas is definitely worse, discard */
+    if(feas && benchres.cSize < winnerResult->cSize && !g_optmode) {
+        return WORSE_RESULT;
+    }
+
+    /* second run, if first run is too short, gives approximate cSpeed + dSpeed */
+    CBENCHMARK(loopDurationC < TIMELOOP_NANOSEC / 10, benchres, tmp, BMK_compressOnly, BMK_iterMode, 1);
+    CBENCHMARK(loopDurationD < TIMELOOP_NANOSEC / 10, benchres, tmp, BMK_decodeOnly,   BMK_iterMode, 1);
+
+    *resultPtr = benchres;
+
+    /* optimistic assumption of benchres */
+    resultMax = benchres;
+    resultMax.cSpeed *= uncertaintyConstantC * VARIANCE;
+    resultMax.dSpeed *= uncertaintyConstantD * VARIANCE;
+
+    /* disregard infeasible results in feas mode */
+    /* disregard if resultMax < winner in infeas mode */
+    if((feas && !feasible(resultMax, target)) ||
+      (!feas && (winnerRS > resultScore(resultMax, buf.srcSize, target)))) {
+        return WORSE_RESULT;
+    }
+
+    CBENCHMARK(loopDurationC < TIMELOOP_NANOSEC, benchres, tmp, BMK_compressOnly, BMK_timeMode, 1);
+    CBENCHMARK(loopDurationD < TIMELOOP_NANOSEC, benchres, tmp, BMK_decodeOnly,   BMK_timeMode, 1);
+
+    *resultPtr = benchres;
+
+    /* compare by resultScore when in infeas */
+    /* compare by compareResultLT when in feas */
+    if((!feas && (resultScore(benchres, buf.srcSize, target) > resultScore(*winnerResult, buf.srcSize, target))) || 
+       (feas && (compareResultLT(*winnerResult, benchres, target, buf.srcSize))) )  { 
+        return BETTER_RESULT; 
+    } else { 
+        return WORSE_RESULT; 
     }
 }
 
-/* 0 for insertion, 1 for no insert */
-/* maintain invariant speedSizeCompare(n, n->next) = SPEED_RESULT */
-static int insertWinner(const winnerInfo_t w, const constraint_t targetConstraints) {
-    BMK_result_t r = w.result;
-    winner_ll_node* cur_node = g_winners;
-    /* first node to insert */
-    if(!feasible(r, targetConstraints)) {
-        return 1;
+#define INFEASIBLE_THRESHOLD 200
+/* Memoized benchmarking, won't benchmark anything which has already been benchmarked before. */
+static int benchMemo(BMK_result_t* resultPtr,
+                const buffers_t buf, const contexts_t ctx, 
+                const paramValues_t cParams,
+                const constraint_t target,
+                BMK_result_t* winnerResult, memoTable_t* const memoTableArray,
+                const int feas) {
+    static int bmcount = 0;
+    int res;
+
+    if(memoTableGet(memoTableArray, cParams) >= INFEASIBLE_THRESHOLD || redundantParams(cParams, target, buf.maxBlockSize)) { return WORSE_RESULT; } 
+
+    res = allBench(resultPtr, buf, ctx, cParams, target, winnerResult, feas);
+
+    if(DEBUG && !(bmcount % 250)) {
+        DISPLAY("Count: %d\n", bmcount);
+        bmcount++;
     }
+    BMK_printWinnerOpt(stdout, CUSTOM_LEVEL, *resultPtr, cParams, target, buf.srcSize);
 
-    if(g_winners == NULL) {
-        winner_ll_node* first_node = malloc(sizeof(winner_ll_node));
-        if(first_node == NULL) {
-            return 1;
-        }
-        first_node->next = NULL;
-        first_node->res = w;
-        g_winners = first_node;
-        return 0;
+    if(res == BETTER_RESULT || feas) {
+        memoTableSet(memoTableArray, cParams, 255); /* what happens if collisions are frequent */
     }
-
-    while(cur_node->next != NULL) {
-        switch(speedSizeCompare(cur_node->res.result, r)) {
-            case WORSE_RESULT:
-            {
-                return 1; /* never insert if better */
-            }
-            case BETTER_RESULT:
-            {
-                winner_ll_node* tmp;
-                cur_node->res = cur_node->next->res;
-                tmp = cur_node->next;
-                cur_node->next = cur_node->next->next;
-                free(tmp);
-                break; 
-            }
-            case SIZE_RESULT:
-            {
-                cur_node = cur_node->next;
-                break;
-            }
-            case SPEED_RESULT: /* insert after first size result, then return */
-            {
-                winner_ll_node* newnode = malloc(sizeof(winner_ll_node));
-                if(newnode == NULL) {
-                    return 1;
-                }
-                newnode->res = cur_node->res;
-                cur_node->res = w;
-                newnode->next = cur_node->next;
-                cur_node->next = newnode;
-                return 0;
-            }
-        } 
-
-    }
-
-    assert(cur_node->next == NULL);
-    switch(speedSizeCompare(cur_node->res.result, r)) {
-        case WORSE_RESULT:
-        {
-            return 1; /* never insert if better */
-        }
-        case BETTER_RESULT:
-        {
-            cur_node->res = w;
-            return 0;
-        }
-        case SIZE_RESULT:
-        {
-            winner_ll_node* newnode = malloc(sizeof(winner_ll_node));
-            if(newnode == NULL) {
-                return 1;
-            }
-            newnode->res = w;
-            newnode->next = NULL;
-            cur_node->next = newnode;
-            return 0;
-        }
-        case SPEED_RESULT: /* insert before first size result, then return */
-        {
-            winner_ll_node* newnode = malloc(sizeof(winner_ll_node));
-            if(newnode == NULL) {
-                return 1;
-            }
-            newnode->res = cur_node->res;
-            cur_node->res = w;
-            newnode->next = cur_node->next;
-            cur_node->next = newnode;
-            return 0;
-        }
-        default: 
-            return 1;
-    } 
+    return res;
 }
 
-/* Writes to f the results of a parameter benchmark */
-/* when used with --optimize, will only print results better than previously discovered */
-static void BMK_printWinner(FILE* f, const U32 cLevel, const BMK_result_t result, const paramValues_t params, const size_t srcSize)
-{
-    char lvlstr[15] = "Custom Level";
-    winnerInfo_t w;
-    w.params = params;
-    w.result = result;
-
-    fprintf(f, "\r%79s\r", "");
-
-    if(cLevel != CUSTOM_LEVEL) {
-        snprintf(lvlstr, 15, "  Level %2u  ", cLevel);
-    }
-
-    if(TIMED) { 
-        const U64 time = UTIL_clockSpanNano(g_time);
-        const U64 minutes = time / (60ULL * TIMELOOP_NANOSEC);
-        fprintf(f, "%1lu:%2lu:%05.2f - ", (unsigned long) minutes / 60,(unsigned long) minutes % 60, (double)(time - minutes * TIMELOOP_NANOSEC * 60ULL)/TIMELOOP_NANOSEC); 
-    }
-
-    fprintf(f, "/* %s */   ", lvlstr);
-    BMK_displayOneResult(f, w, srcSize);
-}
-
-static void BMK_printWinnerOpt(FILE* f, const U32 cLevel, const BMK_result_t result, const paramValues_t params, const constraint_t targetConstraints, const size_t srcSize)
-{
-    /* global winner used for constraints */
-                                    /* cSize, cSpeed, dSpeed, cMem */
-    static winnerInfo_t g_winner = { { (size_t)-1LL, 0, 0, (size_t)-1LL }, { { PARAM_UNSET, PARAM_UNSET, PARAM_UNSET, PARAM_UNSET, PARAM_UNSET, PARAM_UNSET, PARAM_UNSET, PARAM_UNSET } } };
-    if(DEBUG || compareResultLT(g_winner.result, result, targetConstraints, srcSize) || g_displayLevel >= 4) {
-        if(DEBUG && compareResultLT(g_winner.result, result, targetConstraints, srcSize)) {
-            DISPLAY("New Winner: \n");
-        }
-
-        if(g_displayLevel >= 2) { BMK_printWinner(f, cLevel, result, params, srcSize); }
-
-        if(compareResultLT(g_winner.result, result, targetConstraints, srcSize)) {
-            if(g_displayLevel >= 1) { BMK_translateAdvancedParams(f, params); }
-            g_winner.result = result;
-            g_winner.params = params;
-        }
-    }  
-
-    if(g_optmode && g_optimizer && (DEBUG || g_displayLevel == 3)) {
-        winnerInfo_t w;
-        winner_ll_node* n;
-        w.result = result;
-        w.params = params;
-        insertWinner(w, targetConstraints);
-
-        if(!DEBUG) { fprintf(f, "\033c"); }
-        fprintf(f, "\n"); 
-        
-        /* the table */
-        fprintf(f, "================================\n");
-        for(n = g_winners; n != NULL; n = n->next) {
-            BMK_displayOneResult(f, n->res, srcSize);
-        }
-        fprintf(f, "================================\n");
-        fprintf(f, "Level Bounds: R: > %.3f AND C: < %.1f MB/s \n\n",
-            (double)srcSize / g_lvltarget.cSize, (double)g_lvltarget.cSpeed / (1 MB));
-
-
-        fprintf(f, "Overall Winner: \n");
-        BMK_displayOneResult(f, g_winner, srcSize);
-        BMK_translateAdvancedParams(f, g_winner.params);
-
-        fprintf(f, "Latest BMK: \n");\
-        BMK_displayOneResult(f, w, srcSize);
-    }
-}
-
-static void BMK_printWinners2(FILE* f, const winnerInfo_t* winners, const size_t srcSize)
-{
-    int cLevel;
-
-    fprintf(f, "\n /* Proposed configurations : */ \n");
-    fprintf(f, "    /* W,  C,  H,  S,  L,  T, strat */ \n");
-
-    for (cLevel=0; cLevel <= NB_LEVELS_TRACKED; cLevel++)
-        BMK_printWinner(f, cLevel, winners[cLevel].result, winners[cLevel].params, srcSize);
-}
-
-
-static void BMK_printWinners(FILE* f, const winnerInfo_t* winners, const size_t srcSize)
-{
-    fseek(f, 0, SEEK_SET);
-    BMK_printWinners2(f, winners, srcSize);
-    fflush(f);
-    BMK_printWinners2(stdout, winners, srcSize);
-}
-
-
 typedef struct {
     U64 cSpeed_min;
     U64 dSpeed_min;
@@ -1437,201 +1760,9 @@ static int BMK_seed(winnerInfo_t* winners, const paramValues_t params,
     return better;
 }
 
-/* bounds check in sanitize too? */
-#define CLAMP(var, lo, hi) {                          \
-    var = MAX(MIN(var, hi), lo);                      \
-}
-
-/* nullified useless params, to ensure count stats */
-/* cleans up params for memoizing / display */
-static paramValues_t sanitizeParams(paramValues_t params)
-{
-    if (params.vals[strt_ind] == ZSTD_fast)
-        params.vals[clog_ind] = 0, params.vals[slog_ind] = 0;
-    if (params.vals[strt_ind] == ZSTD_dfast)
-        params.vals[slog_ind] = 0;
-    if (params.vals[strt_ind] != ZSTD_btopt && params.vals[strt_ind] != ZSTD_btultra && params.vals[strt_ind] != ZSTD_fast)
-        params.vals[tlen_ind] = 0;
-
-    return params;
-}
-
-/* return: new length */
-/* keep old array, will need if iter over strategy. */
-/* prunes useless params */
-static size_t sanitizeVarArray(varInds_t* varNew, const size_t varLength, const varInds_t* varArray, const ZSTD_strategy strat) {
-    size_t i, j = 0;
-    for(i = 0; i < varLength; i++) {
-        if( !((varArray[i] == clog_ind && strat == ZSTD_fast)
-            || (varArray[i] == slog_ind && strat == ZSTD_fast)
-            || (varArray[i] == slog_ind && strat == ZSTD_dfast) 
-            || (varArray[i] == tlen_ind && strat != ZSTD_btopt && strat != ZSTD_btultra && strat != ZSTD_fast))) {
-            varNew[j] = varArray[i];
-            j++;
-        }
-    }
-    return j;
-
-}
-
-/* res should be NUM_PARAMS size */
-/* constructs varArray from paramValues_t style parameter */
-/* pass in using dict. */
-static size_t variableParams(const paramValues_t paramConstraints, varInds_t* res, const int usingDictionary) {
-    varInds_t i;
-    size_t j = 0;
-    for(i = 0; i < NUM_PARAMS; i++) {
-        if(paramConstraints.vals[i] == PARAM_UNSET) {
-            if(i == fadt_ind && !usingDictionary) continue; /* don't use fadt if no dictionary */
-            res[j] = i; j++;
-        }
-    }
-    return j;
-}
-
-/* amt will probably always be \pm 1? */
-/* slight change from old paramVariation, targetLength can only take on powers of 2 now (999 ~= 1024?) */
-/* take max/min bounds into account as well? */
-static void paramVaryOnce(const varInds_t paramIndex, const int amt, paramValues_t* ptr) {
-    ptr->vals[paramIndex] = rangeMap(paramIndex, invRangeMap(paramIndex, ptr->vals[paramIndex]) + amt);
-}
-
-/* varies ptr by nbChanges respecting varyParams*/
-static void paramVariation(paramValues_t* ptr, memoTable_t* mtAll, const U32 nbChanges)
-{
-    paramValues_t p;
-    U32 validated = 0;
-    while (!validated) {
-        U32 i;
-        p = *ptr;
-        for (i = 0 ; i < nbChanges ; i++) {
-            const U32 changeID = (U32)FUZ_rand(&g_rand) % (mtAll[p.vals[strt_ind]].varLen << 1);
-            paramVaryOnce(mtAll[p.vals[strt_ind]].varArray[changeID >> 1], ((changeID & 1) << 1) - 1, &p);
-        }
-        validated = paramValid(p);
-    }
-    *ptr = p;
-}
-
-/* length of memo table given free variables */
-static size_t memoTableLen(const varInds_t* varyParams, const size_t varyLen) {
-    size_t arrayLen = 1;
-    size_t i;
-    for(i = 0; i < varyLen; i++) {
-        if(varyParams[i] == strt_ind) continue; /* strategy separated by table */
-        arrayLen *= rangetable[varyParams[i]];
-    }
-    return arrayLen;
-}
-
-/* returns unique index in memotable of compression parameters */
-static unsigned memoTableIndDirect(const paramValues_t* ptr, const varInds_t* varyParams, const size_t varyLen) {
-    size_t i;
-    unsigned ind = 0;
-    for(i = 0; i < varyLen; i++) {
-        varInds_t v = varyParams[i];
-        if(v == strt_ind) continue; /* exclude strategy from memotable */
-        ind *= rangetable[v]; ind += (unsigned)invRangeMap(v, ptr->vals[v]);
-    }
-    return ind;
-}
-
-static size_t memoTableGet(const memoTable_t* memoTableArray, const paramValues_t p) {
-    const memoTable_t mt = memoTableArray[p.vals[strt_ind]];
-    switch(mt.tableType) {
-        case directMap:
-            return mt.table[memoTableIndDirect(&p, mt.varArray, mt.varLen)];
-        case xxhashMap:
-            return mt.table[(XXH64(&p.vals, sizeof(U32) * NUM_PARAMS, 0) >> 3) % mt.tableLen];
-        case noMemo:
-            return 0;
-    }
-    return 0; /* should never happen, stop compiler warnings */
-}
-
-static void memoTableSet(const memoTable_t* memoTableArray, const paramValues_t p, const BYTE value) {
-    const memoTable_t mt = memoTableArray[p.vals[strt_ind]];
-    switch(mt.tableType) {
-        case directMap:
-            mt.table[memoTableIndDirect(&p, mt.varArray, mt.varLen)] = value; break;
-        case xxhashMap:
-            mt.table[(XXH64(&p.vals, sizeof(U32) * NUM_PARAMS, 0) >> 3) % mt.tableLen] = value; break;
-        case noMemo:
-            break;
-    }
-}
-
-/* frees all allocated memotables */
-static void freeMemoTableArray(memoTable_t* const mtAll) {
-    int i;
-    if(mtAll == NULL) { return; }
-    for(i = 1; i <= (int)ZSTD_btultra; i++) {
-        free(mtAll[i].table);
-    }
-    free(mtAll);
-}
-
-/* inits memotables for all (including mallocs), all strategies */
-/* takes unsanitized varyParams */
-static memoTable_t* createMemoTableArray(const paramValues_t p, const varInds_t* const varyParams, const size_t varyLen, const U32 memoTableLog) {
-    memoTable_t* mtAll = (memoTable_t*)calloc(sizeof(memoTable_t),(ZSTD_btultra + 1));
-    ZSTD_strategy i, stratMin = ZSTD_fast, stratMax = ZSTD_btultra;
-    
-    if(mtAll == NULL) {
-        return NULL;
-    }
-
-    for(i = 1; i <= (int)ZSTD_btultra; i++) {
-        mtAll[i].varLen = sanitizeVarArray(mtAll[i].varArray, varyLen, varyParams, i);
-    }
-
-    /* no memoization */
-    if(memoTableLog == 0) {
-        for(i = 1; i <= (int)ZSTD_btultra; i++) {
-            mtAll[i].tableType = noMemo;
-            mtAll[i].table = NULL;
-            mtAll[i].tableLen = 0;
-        }
-        return mtAll;
-    }
-
-    
-    if(p.vals[strt_ind] != PARAM_UNSET) {
-        stratMin = p.vals[strt_ind];
-        stratMax = p.vals[strt_ind];
-    }
-
-
-    for(i = stratMin; i <= stratMax; i++) {
-        size_t mtl = memoTableLen(mtAll[i].varArray, mtAll[i].varLen);
-        mtAll[i].tableType = directMap;
-
-        if(memoTableLog != PARAM_UNSET && mtl > (1ULL << memoTableLog)) { /* use hash table */ /* provide some option to only use hash tables? */
-            mtAll[i].tableType = xxhashMap;
-            mtl = (1ULL << memoTableLog);
-        }
-
-        mtAll[i].table = (BYTE*)calloc(sizeof(BYTE), mtl);
-        mtAll[i].tableLen = mtl;
-
-        if(mtAll[i].table == NULL) {
-            freeMemoTableArray(mtAll);
-            return NULL;
-        }
-    }
-    
-    return mtAll;
-}
-
-static paramValues_t overwriteParams(paramValues_t base, const paramValues_t mask) {
-    U32 i;
-    for(i = 0; i < NUM_PARAMS; i++) {
-        if(mask.vals[i] != PARAM_UNSET) {
-            base.vals[i] = mask.vals[i];
-        }
-    }
-    return base;
-}
+/*-************************************
+*  Compression Level Table Generation Functions
+**************************************/
 
 #define PARAMTABLELOG   25
 #define PARAMTABLESIZE (1<vals[strt_ind] = st;
-    for(j = 0; j < mt.tableLen; j++) {
-        int i;
-        for(i = 0; i < NUM_PARAMS; i++) {
-            varInds_t v = mt.varArray[i];
-            if(v == strt_ind) continue; 
-            pc->vals[v] = rangeMap(v, FUZ_rand(&g_rand) % rangetable[v]);
-        }
-
-        if(!(memoTableGet(memoTableArray, *pc))) break; /* only pick unpicked params. */
-    }
-}
-
-/* Completely random parameter selection */
-static paramValues_t randomParams(void)
-{
-    varInds_t v; paramValues_t p;
-    for(v = 0; v <= NUM_PARAMS; v++) {
-        p.vals[v] = rangeMap(v, FUZ_rand(&g_rand) % rangetable[v]);
-    }
-    return p;
-}
-
 static void BMK_selectRandomStart(
                        FILE* f, winnerInfo_t* winners,
                        const buffers_t buf, const contexts_t ctx)
@@ -1764,6 +1866,10 @@ static void BMK_benchFullTable(const buffers_t buf, const contexts_t ctx)
     fclose(f);
 }
 
+/*-************************************
+*  Single Benchmark Functions
+**************************************/
+
 static int benchOnce(const buffers_t buf, const contexts_t ctx) {
     BMK_result_t testResult;
 
@@ -1862,113 +1968,10 @@ int benchFiles(const char** fileNamesTable, int nbFiles, const char* dictFileNam
     return ret;
 }
 
-#define CBENCHMARK(conditional, resultvar, tmpret, mode, loopmode, sec) {                                       \
-    if(conditional) {                                                                                           \
-        BMK_return_t tmpret = BMK_benchMemInvertible(buf, ctx, BASE_CLEVEL, &cParams, mode, loopmode, sec);     \
-        if(tmpret.error) { DEBUGOUTPUT("Benchmarking failed\n"); return ERROR_RESULT; }                         \
-        if(mode != BMK_decodeOnly)  {                                                                           \
-            resultvar.cSpeed = tmpret.result.cSpeed;                                                            \
-            resultvar.cSize = tmpret.result.cSize;                                                              \
-            resultvar.cMem = tmpret.result.cMem;                                                                \
-        }                                                                                                       \
-        if(mode != BMK_compressOnly) { resultvar.dSpeed = tmpret.result.dSpeed; }                               \
-    }                                                                                                           \
-}
 
-/* Benchmarking which stops when we are sufficiently sure the solution is infeasible / worse than the winner */
-#define VARIANCE 1.2 
-static int allBench(BMK_result_t* resultPtr,
-                const buffers_t buf, const contexts_t ctx,
-                const paramValues_t cParams,
-                const constraint_t target,
-                BMK_result_t* winnerResult, int feas) {
-    BMK_result_t resultMax, benchres;
-    U64 loopDurationC = 0, loopDurationD = 0;
-    double uncertaintyConstantC = 3., uncertaintyConstantD = 3.;
-    double winnerRS;
-    /* initial benchmarking, gives exact ratio and memory, warms up future runs */
-    CBENCHMARK(1, benchres, tmp, BMK_both, BMK_iterMode, 1);
-
-    winnerRS = resultScore(*winnerResult, buf.srcSize, target);
-    DEBUGOUTPUT("WinnerScore: %f\n ", winnerRS);
-
-    *resultPtr = benchres;
-
-    /* calculate uncertainty in compression / decompression runs */
-    if(benchres.cSpeed) {
-        loopDurationC = ((buf.srcSize * TIMELOOP_NANOSEC) / benchres.cSpeed); 
-        uncertaintyConstantC = ((loopDurationC + (double)(2 * g_clockGranularity))/loopDurationC); 
-    }
-
-    if(benchres.dSpeed) {
-        loopDurationD = ((buf.srcSize * TIMELOOP_NANOSEC) / benchres.dSpeed); 
-        uncertaintyConstantD = ((loopDurationD + (double)(2 * g_clockGranularity))/loopDurationD);  
-    }
-
-    /* anything with worse ratio in feas is definitely worse, discard */
-    if(feas && benchres.cSize < winnerResult->cSize && !g_optmode) {
-        return WORSE_RESULT;
-    }
-
-    /* second run, if first run is too short, gives approximate cSpeed + dSpeed */
-    CBENCHMARK(loopDurationC < TIMELOOP_NANOSEC / 10, benchres, tmp, BMK_compressOnly, BMK_iterMode, 1);
-    CBENCHMARK(loopDurationD < TIMELOOP_NANOSEC / 10, benchres, tmp, BMK_decodeOnly,   BMK_iterMode, 1);
-
-    *resultPtr = benchres;
-
-    /* optimistic assumption of benchres */
-    resultMax = benchres;
-    resultMax.cSpeed *= uncertaintyConstantC * VARIANCE;
-    resultMax.dSpeed *= uncertaintyConstantD * VARIANCE;
-
-    /* disregard infeasible results in feas mode */
-    /* disregard if resultMax < winner in infeas mode */
-    if((feas && !feasible(resultMax, target)) ||
-      (!feas && (winnerRS > resultScore(resultMax, buf.srcSize, target)))) {
-        return WORSE_RESULT;
-    }
-
-    CBENCHMARK(loopDurationC < TIMELOOP_NANOSEC, benchres, tmp, BMK_compressOnly, BMK_timeMode, 1);
-    CBENCHMARK(loopDurationD < TIMELOOP_NANOSEC, benchres, tmp, BMK_decodeOnly,   BMK_timeMode, 1);
-
-    *resultPtr = benchres;
-
-    /* compare by resultScore when in infeas */
-    /* compare by compareResultLT when in feas */
-    if((!feas && (resultScore(benchres, buf.srcSize, target) > resultScore(*winnerResult, buf.srcSize, target))) || 
-       (feas && (compareResultLT(*winnerResult, benchres, target, buf.srcSize))) )  { 
-        return BETTER_RESULT; 
-    } else { 
-        return WORSE_RESULT; 
-    }
-}
-
-#define INFEASIBLE_THRESHOLD 200
-/* Memoized benchmarking, won't benchmark anything which has already been benchmarked before. */
-static int benchMemo(BMK_result_t* resultPtr,
-                const buffers_t buf, const contexts_t ctx, 
-                const paramValues_t cParams,
-                const constraint_t target,
-                BMK_result_t* winnerResult, memoTable_t* const memoTableArray,
-                const int feas) {
-    static int bmcount = 0;
-    int res;
-
-    if(memoTableGet(memoTableArray, cParams) >= INFEASIBLE_THRESHOLD || redundantParams(cParams, target, buf.maxBlockSize)) { return WORSE_RESULT; } 
-
-    res = allBench(resultPtr, buf, ctx, cParams, target, winnerResult, feas);
-
-    if(DEBUG && !(bmcount % 250)) {
-        DISPLAY("Count: %d\n", bmcount);
-        bmcount++;
-    }
-    BMK_printWinnerOpt(stdout, CUSTOM_LEVEL, *resultPtr, cParams, target, buf.srcSize);
-
-    if(res == BETTER_RESULT || feas) {
-        memoTableSet(memoTableArray, cParams, 255); /* what happens if collisions are frequent */
-    }
-    return res;
-}
+/*-************************************
+*  Local Optimization Functions
+**************************************/
 
 /* One iteration of hill climbing. Specifically, it first tries all 
  * valid parameter configurations w/ manhattan distance 1 and picks the best one
@@ -2368,6 +2371,24 @@ _cleanUp:
     return ret;
 }
 
+/*-************************************
+*  CLI parsing functions
+**************************************/
+
+/** longCommandWArg() :
+ *  check if *stringPtr is the same as longCommand.
+ *  If yes, @return 1 and advances *stringPtr to the position which immediately follows longCommand.
+ * @return 0 and doesn't modify *stringPtr otherwise.
+ * from zstdcli.c
+ */
+static unsigned longCommandWArg(const char** stringPtr, const char* longCommand)
+{
+    size_t const comSize = strlen(longCommand);
+    int const result = !strncmp(*stringPtr, longCommand, comSize);
+    if (result) *stringPtr += comSize;
+    return result;
+}
+
 static void errorOut(const char* msg)
 {
     DISPLAY("%s \n", msg); exit(1);
@@ -2474,6 +2495,10 @@ static int parse_params(const char** argptr, paramValues_t* pv) {
     return matched;
 }
 
+/*-************************************
+*  Main
+**************************************/
+
 int main(int argc, const char** argv)
 {
     int i,

From 239e114d62c2a1d3c73ac05534a18d8583b9654d Mon Sep 17 00:00:00 2001
From: George Lu 
Date: Wed, 15 Aug 2018 15:01:03 -0700
Subject: [PATCH 178/372] prune comments

---
 tests/paramgrill.c | 14 +++-----------
 1 file changed, 3 insertions(+), 11 deletions(-)

diff --git a/tests/paramgrill.c b/tests/paramgrill.c
index 7ebcdeec0..e94eead0d 100644
--- a/tests/paramgrill.c
+++ b/tests/paramgrill.c
@@ -439,9 +439,6 @@ static paramValues_t overwriteParams(paramValues_t base, const paramValues_t mas
     return base;
 }
 
-/* amt will probably always be \pm 1? */
-/* slight change from old paramVariation, targetLength can only take on powers of 2 now (999 ~= 1024?) */
-/* take max/min bounds into account as well? */
 static void paramVaryOnce(const varInds_t paramIndex, const int amt, paramValues_t* ptr) {
     ptr->vals[paramIndex] = rangeMap(paramIndex, invRangeMap(paramIndex, ptr->vals[paramIndex]) + amt);
 }
@@ -610,7 +607,6 @@ static void optimizerAdjustInput(paramValues_t* pc, const size_t maxBlockSize) {
     }
 }
 
-/* what about low something like clog vs hlog in lvl 1?  */
 static int redundantParams(const paramValues_t paramValues, const constraint_t target, const size_t maxBlockSize) {
     return 
        (ZSTD_estimateCStreamSize_usingCParams(pvalsToCParams(paramValues)) > (size_t)target.cMem) /* Uses too much memory */
@@ -684,7 +680,6 @@ static void BMK_printWinner(FILE* f, const U32 cLevel, const BMK_result_t result
 
 /* comparison function: */
 /* strictly better, strictly worse, equal, speed-side adv, size-side adv */
-//Maybe use compress_only for benchmark first run?
 #define WORSE_RESULT 0
 #define BETTER_RESULT 1
 #define ERROR_RESULT 2
@@ -1385,7 +1380,7 @@ static void randomConstrainedParams(paramValues_t* pc, const memoTable_t* memoTa
 /* BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, fileSizes, nbFiles, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); */
 /* nbSeconds used in same way as in BMK_advancedParams_t, as nbIters when in iterMode */
 
-/* if in decodeOnly, then srcPtr's will be compressed blocks, and uncompressedBlocks will be written to dstPtrs? */
+/* if in decodeOnly, then srcPtr's will be compressed blocks, and uncompressedBlocks will be written to dstPtrs */
 /* dictionary nullable, nothing else though. */
 static BMK_return_t BMK_benchMemInvertible(const buffers_t buf, const contexts_t ctx, 
                         const int cLevel, const paramValues_t* comprParams,
@@ -2088,11 +2083,9 @@ static winnerInfo_t climbOnce(const constraint_t target,
 
 /* Optimizes for a fixed strategy */
 
-/* flexible parameters: iterations of (failed?) climbing (or if we do non-random, maybe this is when everything is close to visitied)
+/* flexible parameters: iterations of failed climbing (or if we do non-random, maybe this is when everything is close to visitied)
    weight more on visit for bad results, less on good results/more on later results / ones with more failures.
    allocate memoTable here. 
-   only real use for paramTarget is to get the fixed values, right? 
-   maybe allow giving it a first init? 
  */
 static winnerInfo_t optimizeFixedStrategy(
     const buffers_t buf, const contexts_t ctx, 
@@ -2233,7 +2226,6 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_
     }
 
     /* use level'ing mode instead of normal target mode */
-    /* Should lvl be parameter-masked here? */
     if(g_optmode) {
         winner.params = cParamsToPVals(ZSTD_getCParams(cLevelOpt, buf.maxBlockSize, ctx.dictSize));
         if(BMK_benchParam(&winner.result, buf, ctx, winner.params)) {
@@ -2247,7 +2239,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_
         g_lvltarget.cSize /= ((double)g_strictness) / 100;
 
         target.cSpeed = (U32)g_lvltarget.cSpeed;  
-        target.dSpeed = (U32)g_lvltarget.dSpeed; //See if this is reasonable. 
+        target.dSpeed = (U32)g_lvltarget.dSpeed; 
 
         BMK_printWinnerOpt(stdout, cLevelOpt, winner.result, winner.params, target, buf.srcSize);
     }

From da55865e47bb8035cbf1675d3d80c61979bfc8d3 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Wed, 15 Aug 2018 16:43:13 -0700
Subject: [PATCH 179/372] ensure dependency for zlib wrapper

---
 Makefile                          | 14 +++++++-------
 zlibWrapper/examples/zwrapbench.c |  6 +++---
 zlibWrapper/gzlib.c               |  4 ++--
 zlibWrapper/gzwrite.c             |  9 ++++++---
 4 files changed, 18 insertions(+), 15 deletions(-)

diff --git a/Makefile b/Makefile
index f8441e60d..d33757603 100644
--- a/Makefile
+++ b/Makefile
@@ -30,8 +30,7 @@ default: lib-release zstd-release
 all: allmost examples manual contrib
 
 .PHONY: allmost
-allmost: allzstd
-	$(MAKE) -C $(ZWRAPDIR) all
+allmost: allzstd zlibwrapper
 
 #skip zwrapper, can't build that on alternate architectures without the proper zlib installed
 .PHONY: allzstd
@@ -44,8 +43,9 @@ all32:
 	$(MAKE) -C $(PRGDIR) zstd32
 	$(MAKE) -C $(TESTDIR) all32
 
-.PHONY: lib lib-release
-lib lib-release:
+.PHONY: lib lib-release libzstd.a
+lib : libzstd.a
+lib lib-release libzstd.a:
 	@$(MAKE) -C $(ZSTDDIR) $@
 
 .PHONY: zstd zstd-release
@@ -59,8 +59,8 @@ zstdmt:
 	cp $(PRGDIR)/zstd$(EXT) ./zstdmt$(EXT)
 
 .PHONY: zlibwrapper
-zlibwrapper:
-	$(MAKE) -C $(ZWRAPDIR) test
+zlibwrapper: libzstd.a
+	$(MAKE) -C $(ZWRAPDIR) all
 
 .PHONY: test
 test: MOREFLAGS += -g -DDEBUGLEVEL=1 -Werror
@@ -353,5 +353,5 @@ bmi32build: clean
 
 staticAnalyze:
 	$(CC) -v
-	CPPFLAGS=-g scan-build --status-bugs -v $(MAKE) all
+	CC=$(CC) CPPFLAGS=-g scan-build --status-bugs -v $(MAKE) all
 endif
diff --git a/zlibWrapper/examples/zwrapbench.c b/zlibWrapper/examples/zwrapbench.c
index a4dfbb6e8..d2d6073f9 100644
--- a/zlibWrapper/examples/zwrapbench.c
+++ b/zlibWrapper/examples/zwrapbench.c
@@ -573,10 +573,10 @@ static size_t BMK_findMaxMem(U64 requiredMem)
     do {
         testmem = (BYTE*)malloc((size_t)requiredMem);
         requiredMem -= step;
-    } while (!testmem);
+    } while (!testmem && requiredMem);   /* do not allocate zero bytes */
 
     free(testmem);
-    return (size_t)(requiredMem);
+    return (size_t)(requiredMem+1);  /* avoid zero */
 }
 
 static void BMK_benchCLevel(void* srcBuffer, size_t benchedSize,
@@ -734,7 +734,7 @@ static void BMK_benchFileTable(const char** fileNamesTable, unsigned nbFiles,
     if ((U64)benchedSize > totalSizeToLoad) benchedSize = (size_t)totalSizeToLoad;
     if (benchedSize < totalSizeToLoad)
         DISPLAY("Not enough memory; testing %u MB only...\n", (U32)(benchedSize >> 20));
-    srcBuffer = malloc(benchedSize);
+    srcBuffer = malloc(benchedSize + !benchedSize);
     if (!srcBuffer) EXM_THROW(12, "not enough memory");
 
     /* Load input buffer */
diff --git a/zlibWrapper/gzlib.c b/zlibWrapper/gzlib.c
index 8235cff4f..3070dd8b4 100644
--- a/zlibWrapper/gzlib.c
+++ b/zlibWrapper/gzlib.c
@@ -111,7 +111,7 @@ local gzFile gz_open(path, fd, mode)
         return NULL;
 
     /* allocate gzFile structure to return */
-    state = (gz_statep)(gz_state*)malloc(sizeof(gz_state));
+    state.state = (gz_state*)malloc(sizeof(gz_state));
     if (state.state == NULL)
         return NULL;
     state.state->size = 0;            /* no buffers allocated yet */
@@ -266,7 +266,7 @@ local gzFile gz_open(path, fd, mode)
     gz_reset(state);
 
     /* return stream */
-    return (gzFile)state.file;
+    return state.file;
 }
 
 /* -- see zlib.h -- */
diff --git a/zlibWrapper/gzwrite.c b/zlibWrapper/gzwrite.c
index d1250b900..21d5f8472 100644
--- a/zlibWrapper/gzwrite.c
+++ b/zlibWrapper/gzwrite.c
@@ -6,6 +6,8 @@
  * For conditions of distribution and use, see http://www.zlib.net/zlib_license.html
  */
 
+#include 
+
 #include "gzguts.h"
 
 /* Local functions */
@@ -24,7 +26,7 @@ local int gz_init(state)
     z_streamp strm = &(state.state->strm);
 
     /* allocate input buffer (double size for gzprintf) */
-    state.state->in = (unsigned char *)malloc(state.state->want << 1);
+    state.state->in = (unsigned char*)malloc(state.state->want << 1);
     if (state.state->in == NULL) {
         gz_error(state, Z_MEM_ERROR, "out of memory");
         return -1;
@@ -33,7 +35,7 @@ local int gz_init(state)
     /* only need output buffer and deflate state if compressing */
     if (!state.state->direct) {
         /* allocate output buffer */
-        state.state->out = (unsigned char *)malloc(state.state->want);
+        state.state->out = (unsigned char*)malloc(state.state->want);
         if (state.state->out == NULL) {
             free(state.state->in);
             gz_error(state, Z_MEM_ERROR, "out of memory");
@@ -284,6 +286,7 @@ z_size_t ZEXPORT gzfwrite(buf, size, nitems, file)
     gz_statep state;
 
     /* get internal structure */
+    assert(size != 0);
     if (file == NULL)
         return 0;
     state = (gz_statep)file;
@@ -294,7 +297,7 @@ z_size_t ZEXPORT gzfwrite(buf, size, nitems, file)
 
     /* compute bytes to read -- error on overflow */
     len = nitems * size;
-    if (size && len / size != nitems) {
+    if (size && (len / size != nitems)) {
         gz_error(state, Z_STREAM_ERROR, "request does not fit in a size_t");
         return 0;
     }

From 5291d9ac31a87b68e717e43bf5b14a71ebaf644d Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Wed, 15 Aug 2018 17:41:44 -0700
Subject: [PATCH 180/372] fix scope of scan-build tests

exclude zlib code
---
 Makefile                         | 11 ++++++-----
 lib/decompress/zstd_decompress.c |  1 +
 2 files changed, 7 insertions(+), 5 deletions(-)

diff --git a/Makefile b/Makefile
index d33757603..e338c5ca1 100644
--- a/Makefile
+++ b/Makefile
@@ -32,7 +32,7 @@ all: allmost examples manual contrib
 .PHONY: allmost
 allmost: allzstd zlibwrapper
 
-#skip zwrapper, can't build that on alternate architectures without the proper zlib installed
+# skip zwrapper, can't build that on alternate architectures without the proper zlib installed
 .PHONY: allzstd
 allzstd: lib
 	$(MAKE) -C $(PRGDIR) all
@@ -44,8 +44,7 @@ all32:
 	$(MAKE) -C $(TESTDIR) all32
 
 .PHONY: lib lib-release libzstd.a
-lib : libzstd.a
-lib lib-release libzstd.a:
+lib lib-release :
 	@$(MAKE) -C $(ZSTDDIR) $@
 
 .PHONY: zstd zstd-release
@@ -59,7 +58,7 @@ zstdmt:
 	cp $(PRGDIR)/zstd$(EXT) ./zstdmt$(EXT)
 
 .PHONY: zlibwrapper
-zlibwrapper: libzstd.a
+zlibwrapper: lib
 	$(MAKE) -C $(ZWRAPDIR) all
 
 .PHONY: test
@@ -351,7 +350,9 @@ bmi32build: clean
 	$(CC) -v
 	CFLAGS="-O3 -mbmi -m32 -Werror" $(MAKE) -C $(TESTDIR) test
 
+# static analyzer test uses clang's scan-build
+# does not analyze zlibWrapper, due to detected issues in zlib source code
 staticAnalyze:
 	$(CC) -v
-	CC=$(CC) CPPFLAGS=-g scan-build --status-bugs -v $(MAKE) all
+	CC=$(CC) CPPFLAGS=-g scan-build --status-bugs -v $(MAKE) allzstd examples contrib
 endif
diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c
index 5d9f0ba0b..8fef7e51b 100644
--- a/lib/decompress/zstd_decompress.c
+++ b/lib/decompress/zstd_decompress.c
@@ -2399,6 +2399,7 @@ static size_t ZSTD_initDDict_internal(ZSTD_DDict* ddict,
     if ((dictLoadMethod == ZSTD_dlm_byRef) || (!dict) || (!dictSize)) {
         ddict->dictBuffer = NULL;
         ddict->dictContent = dict;
+        if (!dict) dictSize = 0;
     } else {
         void* const internalBuffer = ZSTD_malloc(dictSize, ddict->cMem);
         ddict->dictBuffer = internalBuffer;

From 31224cc126f23d2b7dbac84940aaedbf55ea477b Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Wed, 15 Aug 2018 17:44:27 -0700
Subject: [PATCH 181/372] added static analyzer tests to travis CI

---
 .travis.yml | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 36df67052..a61b82876 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -20,23 +20,25 @@ matrix:
     - env: Cmd='make gcc8install && CC=gcc-8 CFLAGS="-Werror -O3" make -j all'
     - env: Cmd='make clang38install && CC=clang-3.8 make clean msan-test-zstd'
 
+    - env: Cmd='make staticAnalyze'
+
     - env: Cmd='make gcc6install && CC=gcc-6 make clean uasan-fuzztest'
     - env: Cmd='make gcc6install libc6install
              && make clean && CC=gcc-6 CFLAGS=-m32 make uasan-fuzztest'
     - env: Cmd='make clang38install && CC=clang-3.8 make clean msan-fuzztest'
     - env: Cmd='make clang38install && CC=clang-3.8 make clean tsan-test-zstream'
 
-    - env: Cmd='make arminstall && make armfuzz'
-    - env: Cmd='make arminstall && make aarch64fuzz'
-    - env: Cmd='make ppcinstall && make ppcfuzz'
-    - env: Cmd='make ppcinstall && make ppc64fuzz'
-
     - env: Cmd='make -j uasanregressiontest
              && make clean && make -j msanregressiontest'
 
     - env: Cmd='make valgrindinstall && make -C tests clean valgrindTest
              && make clean && make -C tests test-fuzzer-stackmode'
 
+    - env: Cmd='make arminstall && make armfuzz'
+    - env: Cmd='make arminstall && make aarch64fuzz'
+    - env: Cmd='make ppcinstall && make ppcfuzz'
+    - env: Cmd='make ppcinstall && make ppc64fuzz'
+
     - env: Cmd='make lz4install && make -C tests test-lz4
              && make clean && make -C tests test-pool
              && make clean && bash tests/libzstd_partial_builds.sh'

From 1515f0bb0d73138dd191a9fd5caad09e9868a8a0 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Thu, 16 Aug 2018 14:40:47 -0700
Subject: [PATCH 182/372] fixed more issues detected by recent version of
 scan-build

test run on Linux
---
 examples/multiple_streaming_compression.c |  3 ++-
 examples/streaming_compression.c          | 22 ++++++++++++++++------
 lib/compress/fse_compress.c               | 22 +++++++++++++++-------
 lib/dictBuilder/divsufsort.c              |  6 +++---
 4 files changed, 36 insertions(+), 17 deletions(-)

diff --git a/examples/multiple_streaming_compression.c b/examples/multiple_streaming_compression.c
index e395aefbc..4308a2e4d 100644
--- a/examples/multiple_streaming_compression.c
+++ b/examples/multiple_streaming_compression.c
@@ -158,7 +158,8 @@ int main(int argc, const char** argv)
     }
 
     freeResources(ress);
-    /* success */
+    free(ofnBuffer);
+
     printf("compressed %i files \n", argc-1);
 
     return 0;
diff --git a/examples/streaming_compression.c b/examples/streaming_compression.c
index f76364d84..9287ff398 100644
--- a/examples/streaming_compression.c
+++ b/examples/streaming_compression.c
@@ -73,7 +73,11 @@ static void compressFile_orDie(const char* fname, const char* outName, int cLeve
     ZSTD_CStream* const cstream = ZSTD_createCStream();
     if (cstream==NULL) { fprintf(stderr, "ZSTD_createCStream() error \n"); exit(10); }
     size_t const initResult = ZSTD_initCStream(cstream, cLevel);
-    if (ZSTD_isError(initResult)) { fprintf(stderr, "ZSTD_initCStream() error : %s \n", ZSTD_getErrorName(initResult)); exit(11); }
+    if (ZSTD_isError(initResult)) {
+        fprintf(stderr, "ZSTD_initCStream() error : %s \n",
+                    ZSTD_getErrorName(initResult));
+        exit(11);
+    }
 
     size_t read, toRead = buffInSize;
     while( (read = fread_orDie(buffIn, toRead, fin)) ) {
@@ -81,7 +85,11 @@ static void compressFile_orDie(const char* fname, const char* outName, int cLeve
         while (input.pos < input.size) {
             ZSTD_outBuffer output = { buffOut, buffOutSize, 0 };
             toRead = ZSTD_compressStream(cstream, &output , &input);   /* toRead is guaranteed to be <= ZSTD_CStreamInSize() */
-            if (ZSTD_isError(toRead)) { fprintf(stderr, "ZSTD_compressStream() error : %s \n", ZSTD_getErrorName(toRead)); exit(12); }
+            if (ZSTD_isError(toRead)) {
+                fprintf(stderr, "ZSTD_compressStream() error : %s \n",
+                                ZSTD_getErrorName(toRead));
+                exit(12);
+            }
             if (toRead > buffInSize) toRead = buffInSize;   /* Safely handle case when `buffInSize` is manually changed to a value < ZSTD_CStreamInSize()*/
             fwrite_orDie(buffOut, output.pos, fout);
         }
@@ -100,15 +108,15 @@ static void compressFile_orDie(const char* fname, const char* outName, int cLeve
 }
 
 
-static const char* createOutFilename_orDie(const char* filename)
+static char* createOutFilename_orDie(const char* filename)
 {
     size_t const inL = strlen(filename);
     size_t const outL = inL + 5;
-    void* outSpace = malloc_orDie(outL);
+    void* const outSpace = malloc_orDie(outL);
     memset(outSpace, 0, outL);
     strcat(outSpace, filename);
     strcat(outSpace, ".zst");
-    return (const char*)outSpace;
+    return (char*)outSpace;
 }
 
 int main(int argc, const char** argv)
@@ -124,8 +132,10 @@ int main(int argc, const char** argv)
 
     const char* const inFilename = argv[1];
 
-    const char* const outFilename = createOutFilename_orDie(inFilename);
+    char* const outFilename = createOutFilename_orDie(inFilename);
     compressFile_orDie(inFilename, outFilename, 1);
 
+    free(outFilename);   /* not strictly required, since program execution stops there,
+                          * but some static analyzer main complain otherwise */
     return 0;
 }
diff --git a/lib/compress/fse_compress.c b/lib/compress/fse_compress.c
index caa90edec..4075db217 100644
--- a/lib/compress/fse_compress.c
+++ b/lib/compress/fse_compress.c
@@ -240,10 +240,12 @@ FSE_writeNCount_generic (void* header, size_t headerBufferSize,
         if (previousIs0) {
             unsigned start = symbol;
             while ((symbol < alphabetSize) && !normalizedCounter[symbol]) symbol++;
+            if (symbol == alphabetSize) break;   /* incorrect distribution */
             while (symbol >= start+24) {
                 start+=24;
                 bitStream += 0xFFFFU << bitCount;
-                if ((!writeIsSafe) && (out > oend-2)) return ERROR(dstSize_tooSmall);   /* Buffer overflow */
+                if ((!writeIsSafe) && (out > oend-2))
+                    return ERROR(dstSize_tooSmall);   /* Buffer overflow */
                 out[0] = (BYTE) bitStream;
                 out[1] = (BYTE)(bitStream>>8);
                 out+=2;
@@ -257,7 +259,8 @@ FSE_writeNCount_generic (void* header, size_t headerBufferSize,
             bitStream += (symbol-start) << bitCount;
             bitCount += 2;
             if (bitCount>16) {
-                if ((!writeIsSafe) && (out > oend - 2)) return ERROR(dstSize_tooSmall);   /* Buffer overflow */
+                if ((!writeIsSafe) && (out > oend - 2))
+                    return ERROR(dstSize_tooSmall);   /* Buffer overflow */
                 out[0] = (BYTE)bitStream;
                 out[1] = (BYTE)(bitStream>>8);
                 out += 2;
@@ -268,7 +271,8 @@ FSE_writeNCount_generic (void* header, size_t headerBufferSize,
             int const max = (2*threshold-1) - remaining;
             remaining -= count < 0 ? -count : count;
             count++;   /* +1 for extra accuracy */
-            if (count>=threshold) count += max;   /* [0..max[ [max..threshold[ (...) [threshold+max 2*threshold[ */
+            if (count>=threshold)
+                count += max;   /* [0..max[ [max..threshold[ (...) [threshold+max 2*threshold[ */
             bitStream += count << bitCount;
             bitCount  += nbBits;
             bitCount  -= (count>=1; }
         }
         if (bitCount>16) {
-            if ((!writeIsSafe) && (out > oend - 2)) return ERROR(dstSize_tooSmall);   /* Buffer overflow */
+            if ((!writeIsSafe) && (out > oend - 2))
+                return ERROR(dstSize_tooSmall);   /* Buffer overflow */
             out[0] = (BYTE)bitStream;
             out[1] = (BYTE)(bitStream>>8);
             out += 2;
@@ -285,11 +290,13 @@ FSE_writeNCount_generic (void* header, size_t headerBufferSize,
             bitCount -= 16;
     }   }
 
-    if (remaining != 1) return ERROR(GENERIC);  /* incorrect normalized distribution */
+    if (remaining != 1)
+        return ERROR(GENERIC);  /* incorrect normalized distribution */
     assert(symbol <= alphabetSize);
 
     /* flush remaining bitStream */
-    if ((!writeIsSafe) && (out > oend - 2)) return ERROR(dstSize_tooSmall);   /* Buffer overflow */
+    if ((!writeIsSafe) && (out > oend - 2))
+        return ERROR(dstSize_tooSmall);   /* Buffer overflow */
     out[0] = (BYTE)bitStream;
     out[1] = (BYTE)(bitStream>>8);
     out+= (bitCount+7) /8;
@@ -298,7 +305,8 @@ FSE_writeNCount_generic (void* header, size_t headerBufferSize,
 }
 
 
-size_t FSE_writeNCount (void* buffer, size_t bufferSize, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog)
+size_t FSE_writeNCount (void* buffer, size_t bufferSize,
+                  const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog)
 {
     if (tableLog > FSE_MAX_TABLELOG) return ERROR(tableLog_tooLarge);   /* Unsupported */
     if (tableLog < FSE_MIN_TABLELOG) return ERROR(GENERIC);   /* Unsupported */
diff --git a/lib/dictBuilder/divsufsort.c b/lib/dictBuilder/divsufsort.c
index 60cceb088..ead922044 100644
--- a/lib/dictBuilder/divsufsort.c
+++ b/lib/dictBuilder/divsufsort.c
@@ -1637,7 +1637,7 @@ construct_SA(const unsigned char *T, int *SA,
             if(0 <= c2) { BUCKET_B(c2, c1) = k - SA; }
             k = SA + BUCKET_B(c2 = c0, c1);
           }
-          assert(k < j);
+          assert(k < j); assert(k != NULL);
           *k-- = s;
         } else {
           assert(((s == 0) && (T[s] == c1)) || (s < 0));
@@ -1701,7 +1701,7 @@ construct_BWT(const unsigned char *T, int *SA,
             if(0 <= c2) { BUCKET_B(c2, c1) = k - SA; }
             k = SA + BUCKET_B(c2 = c0, c1);
           }
-          assert(k < j);
+          assert(k < j); assert(k != NULL);
           *k-- = s;
         } else if(s != 0) {
           *j = ~s;
@@ -1785,7 +1785,7 @@ construct_BWT_indexes(const unsigned char *T, int *SA,
             if(0 <= c2) { BUCKET_B(c2, c1) = k - SA; }
             k = SA + BUCKET_B(c2 = c0, c1);
           }
-          assert(k < j);
+          assert(k < j); assert(k != NULL);
           *k-- = s;
         } else if(s != 0) {
           *j = ~s;

From 36d6165a2d75ab45ddec7da1f681e4153d3f8153 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Thu, 16 Aug 2018 15:41:56 -0700
Subject: [PATCH 183/372] Makefile: added variable SCANBUILD

so that a different version of scan-build can be selected
---
 Makefile                                                | 3 ++-
 contrib/seekable_format/examples/seekable_compression.c | 7 ++++---
 lib/decompress/zstd_decompress.c                        | 1 +
 lib/legacy/zstd_v05.c                                   | 1 +
 lib/legacy/zstd_v06.c                                   | 1 +
 5 files changed, 9 insertions(+), 4 deletions(-)

diff --git a/Makefile b/Makefile
index e338c5ca1..853ceee93 100644
--- a/Makefile
+++ b/Makefile
@@ -352,7 +352,8 @@ bmi32build: clean
 
 # static analyzer test uses clang's scan-build
 # does not analyze zlibWrapper, due to detected issues in zlib source code
+staticAnalyze: SCANBUILD ?= scan-build
 staticAnalyze:
 	$(CC) -v
-	CC=$(CC) CPPFLAGS=-g scan-build --status-bugs -v $(MAKE) allzstd examples contrib
+	CC=$(CC) CPPFLAGS=-g $(SCANBUILD) --status-bugs -v $(MAKE) allzstd examples contrib
 endif
diff --git a/contrib/seekable_format/examples/seekable_compression.c b/contrib/seekable_format/examples/seekable_compression.c
index 9485bf26f..9a331a895 100644
--- a/contrib/seekable_format/examples/seekable_compression.c
+++ b/contrib/seekable_format/examples/seekable_compression.c
@@ -101,7 +101,7 @@ static void compressFile_orDie(const char* fname, const char* outName, int cLeve
     free(buffOut);
 }
 
-static const char* createOutFilename_orDie(const char* filename)
+static char* createOutFilename_orDie(const char* filename)
 {
     size_t const inL = strlen(filename);
     size_t const outL = inL + 5;
@@ -109,7 +109,7 @@ static const char* createOutFilename_orDie(const char* filename)
     memset(outSpace, 0, outL);
     strcat(outSpace, filename);
     strcat(outSpace, ".zst");
-    return (const char*)outSpace;
+    return (char*)outSpace;
 }
 
 int main(int argc, const char** argv) {
@@ -124,8 +124,9 @@ int main(int argc, const char** argv) {
     {   const char* const inFileName = argv[1];
         unsigned const frameSize = (unsigned)atoi(argv[2]);
 
-        const char* const outFileName = createOutFilename_orDie(inFileName);
+        char* const outFileName = createOutFilename_orDie(inFileName);
         compressFile_orDie(inFileName, outFileName, 5, frameSize);
+        free(outFileName);
     }
 
     return 0;
diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c
index 8fef7e51b..7c4769cd8 100644
--- a/lib/decompress/zstd_decompress.c
+++ b/lib/decompress/zstd_decompress.c
@@ -542,6 +542,7 @@ size_t ZSTD_getcBlockSize(const void* src, size_t srcSize,
 static size_t ZSTD_copyRawBlock(void* dst, size_t dstCapacity,
                           const void* src, size_t srcSize)
 {
+    if (dst==NULL) return ERROR(dstSize_tooSmall);
     if (srcSize > dstCapacity) return ERROR(dstSize_tooSmall);
     memcpy(dst, src, srcSize);
     return srcSize;
diff --git a/lib/legacy/zstd_v05.c b/lib/legacy/zstd_v05.c
index 89a5fe7e6..65f07a3b5 100644
--- a/lib/legacy/zstd_v05.c
+++ b/lib/legacy/zstd_v05.c
@@ -2846,6 +2846,7 @@ size_t ZSTDv05_getcBlockSize(const void* src, size_t srcSize, blockProperties_t*
 
 static size_t ZSTDv05_copyRawBlock(void* dst, size_t maxDstSize, const void* src, size_t srcSize)
 {
+    if (dst==NULL) return ERROR(dstSize_tooSmall);
     if (srcSize > maxDstSize) return ERROR(dstSize_tooSmall);
     memcpy(dst, src, srcSize);
     return srcSize;
diff --git a/lib/legacy/zstd_v06.c b/lib/legacy/zstd_v06.c
index d8de77ca9..ca982c35e 100644
--- a/lib/legacy/zstd_v06.c
+++ b/lib/legacy/zstd_v06.c
@@ -3041,6 +3041,7 @@ size_t ZSTDv06_getcBlockSize(const void* src, size_t srcSize, blockProperties_t*
 
 static size_t ZSTDv06_copyRawBlock(void* dst, size_t dstCapacity, const void* src, size_t srcSize)
 {
+    if (dst==NULL) return ERROR(dstSize_tooSmall);
     if (srcSize > dstCapacity) return ERROR(dstSize_tooSmall);
     memcpy(dst, src, srcSize);
     return srcSize;

From 8175b28f03bc49a269c5481f8402b52b712d644c Mon Sep 17 00:00:00 2001
From: George Lu 
Date: Thu, 16 Aug 2018 16:13:02 -0700
Subject: [PATCH 184/372] Fix negative lvl display value

Also fix synthetic benchmark parameter setting
---
 tests/paramgrill.c | 19 +++++++++----------
 1 file changed, 9 insertions(+), 10 deletions(-)

diff --git a/tests/paramgrill.c b/tests/paramgrill.c
index e94eead0d..f42afe403 100644
--- a/tests/paramgrill.c
+++ b/tests/paramgrill.c
@@ -655,7 +655,7 @@ static void BMK_displayOneResult(FILE* f, winnerInfo_t res, const size_t srcSize
 
 /* Writes to f the results of a parameter benchmark */
 /* when used with --optimize, will only print results better than previously discovered */
-static void BMK_printWinner(FILE* f, const U32 cLevel, const BMK_result_t result, const paramValues_t params, const size_t srcSize)
+static void BMK_printWinner(FILE* f, const int cLevel, const BMK_result_t result, const paramValues_t params, const size_t srcSize)
 {
     char lvlstr[15] = "Custom Level";
     winnerInfo_t w;
@@ -665,7 +665,7 @@ static void BMK_printWinner(FILE* f, const U32 cLevel, const BMK_result_t result
     fprintf(f, "\r%79s\r", "");
 
     if(cLevel != CUSTOM_LEVEL) {
-        snprintf(lvlstr, 15, "  Level %2u  ", cLevel);
+        snprintf(lvlstr, 15, "  Level %2d  ", cLevel);
     }
 
     if(TIMED) { 
@@ -1865,8 +1865,9 @@ static void BMK_benchFullTable(const buffers_t buf, const contexts_t ctx)
 *  Single Benchmark Functions
 **************************************/
 
-static int benchOnce(const buffers_t buf, const contexts_t ctx) {
+static int benchOnce(const buffers_t buf, const contexts_t ctx, const int cLevel) {
     BMK_result_t testResult;
+    g_params = adjustParams(overwriteParams(cParamsToPVals(ZSTD_getCParams(cLevel, buf.maxBlockSize, ctx.dictSize)), g_params), buf.maxBlockSize, ctx.dictSize);
 
     if(BMK_benchParam(&testResult, buf, ctx, g_params)) {
         DISPLAY("Error during benchmarking\n");
@@ -1878,7 +1879,7 @@ static int benchOnce(const buffers_t buf, const contexts_t ctx) {
     return 0;
 }
 
-static int benchSample(double compressibility)
+static int benchSample(double compressibility, int cLevel)
 {
     const char* const name = "Sample 10MB";
     size_t const benchedSize = 10 MB;
@@ -1912,7 +1913,7 @@ static int benchSample(double compressibility)
     DISPLAY("using %s %i%%: \n", name, (int)(compressibility*100));
 
     if(g_singleRun) {
-        ret = benchOnce(buf, ctx);
+        ret = benchOnce(buf, ctx, cLevel);
     } else {
         BMK_benchFullTable(buf, ctx);
     }
@@ -1926,7 +1927,7 @@ static int benchSample(double compressibility)
 /* benchFiles() :
  * note: while this function takes a table of filenames,
  * in practice, only the first filename will be used */
-int benchFiles(const char** fileNamesTable, int nbFiles, const char* dictFileName, int cLevel)
+int benchFiles(const char** fileNamesTable, int nbFiles, const char* dictFileName, const int cLevel)
 {
     buffers_t buf;
     contexts_t ctx;
@@ -1950,10 +1951,8 @@ int benchFiles(const char** fileNamesTable, int nbFiles, const char* dictFileNam
         DISPLAY("using %d Files : \n", nbFiles);
     }
 
-    g_params = adjustParams(overwriteParams(cParamsToPVals(ZSTD_getCParams(cLevel, buf.maxBlockSize, ctx.dictSize)), g_params), buf.maxBlockSize, ctx.dictSize);
-
     if(g_singleRun) {
-        ret = benchOnce(buf, ctx);
+        ret = benchOnce(buf, ctx, cLevel);
     } else {
         BMK_benchFullTable(buf, ctx);
     }
@@ -2731,7 +2730,7 @@ int main(int argc, const char** argv)
             DISPLAY("Optimizer Expects File\n");
             return 1;
         } else {
-            result = benchSample(compressibility);
+            result = benchSample(compressibility, cLevelRun);
         }
     } else {
         if(seperateFiles) {

From 3959ba15e633776456ebd3ed6bb96db5d775d1cd Mon Sep 17 00:00:00 2001
From: George Lu 
Date: Thu, 16 Aug 2018 17:22:29 -0700
Subject: [PATCH 185/372] Clarify README

---
 tests/README.md | 28 +++++++++++++++++-----------
 1 file changed, 17 insertions(+), 11 deletions(-)

diff --git a/tests/README.md b/tests/README.md
index c1128571a..f28766bd1 100644
--- a/tests/README.md
+++ b/tests/README.md
@@ -104,34 +104,40 @@ Full list of arguments
     t# - targetLength
     S# - strategy
     L# - level
- --zstd=      : Single run, parameter selection syntax same as zstdcli. 
-                When invoked with --optimize, this represents the sample to exceed. 
+ --zstd=      : Single run, parameter selection syntax same as zstdcli with more parameters
+                    (Added forceAttachDictionary / fadt) 
+                    When invoked with --optimize, this represents the sample to exceed. 
  --optimize=  : find parameters to maximize compression ratio given parameters
-                Can use all --zstd= commands to constrain the type of solution found in addition to the following constraints
+                    Can use all --zstd= commands to constrain the type of solution found in addition to the following constraints
     cSpeed=   : Minimum compression speed
     dSpeed=   : Minimum decompression speed
     cMem=     : Maximum compression memory
     lvl=      : Searches for solutions which are strictly better than that compression lvl in ratio and cSpeed, 
     stc=      : When invoked with lvl=, represents percentage slack in ratio/cSpeed allowed for a solution to be considered (Default 100%)
               : In normal operation, represents percentage slack in choosing viable starting strategy selection in choosing the default parameters
-                (Lower value will begin with stronger strategies) (Default 90%)
+                    (Lower value will begin with stronger strategies) (Default 90%)
     speedRatio=   (accepts decimals)
               : determines value of gains in speed vs gains in ratio
-                when determining overall winner (default 5 (1% ratio = 5% speed)).
+                    when determining overall winner (default 5 (1% ratio = 5% speed)).
     tries=    : Maximum number of random restarts on a single strategy before switching (Default 5)
-                Higher values will make optimizer run longer, more chances to find better solution.
-    memLog    : Limits the log of the size of each memotable (1 per strategy). Setting memLog = 0 turns off memoization 
+                    Higher values will make optimizer run longer, more chances to find better solution.
+    memLog    : Limits the log of the size of each memotable (1 per strategy). Will use hash tables when state space is larger than max size. 
+                    Setting memLog = 0 turns off memoization 
  --display=   : specifiy which parameters are included in the output
-                can use all --zstd parameter names and 'cParams' as a shorthand for all parameters used in ZSTD_compressionParameters 
-                (Default: display all params available)
-
- -P#          : generated sample compressibility 
+                    can use all --zstd parameter names and 'cParams' as a shorthand for all parameters used in ZSTD_compressionParameters 
+                    (Default: display all params available)
+ -P#          : generated sample compressibility (when no file is provided)
  -t#          : Caps runtime of operation in seconds (default : 99999 seconds (about 27 hours )) 
  -v           : Prints Benchmarking output
  -D           : Next argument dictionary file
  -s           : Benchmark all files separately
  -q           : Quiet, repeat for more quiet
+                  -q Prints parameters + results whenever a new best is found
+                  -qq Only prints parameters whenever a new best is found, prints final parameters + results
+                  -qqq Only print final parameters + results
+                  -qqqq Only prints final parameter set in the form --zstd=
  -v           : Verbose, cancels quiet, repeat for more volume
+                  -v Prints all candidate parameters and results
 
 ```
  Any inputs afterwards are treated as files to benchmark.

From e400a86f1727918dbc2f073ce74025a0726eda8e Mon Sep 17 00:00:00 2001
From: Timo Gurr 
Date: Fri, 17 Aug 2018 13:31:55 +0200
Subject: [PATCH 186/372] Use GNUInstallDirs DOCDIR for installing the manual

---
 build/cmake/contrib/gen_html/CMakeLists.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/build/cmake/contrib/gen_html/CMakeLists.txt b/build/cmake/contrib/gen_html/CMakeLists.txt
index 958e9d173..c4062d475 100644
--- a/build/cmake/contrib/gen_html/CMakeLists.txt
+++ b/build/cmake/contrib/gen_html/CMakeLists.txt
@@ -27,4 +27,4 @@ ADD_CUSTOM_TARGET(zstd_manual.html ALL
                   ${GENHTML_BINARY} "${LIBVERSION}" "${LIBRARY_DIR}/zstd.h" "${PROJECT_BINARY_DIR}/zstd_manual.html"
                   DEPENDS gen_html COMMENT "Update zstd manual")
 
-INSTALL(FILES "${PROJECT_BINARY_DIR}/zstd_manual.html" DESTINATION "${CMAKE_INSTALL_PREFIX}/${DOC_INSTALL_DIR}")
+INSTALL(FILES "${PROJECT_BINARY_DIR}/zstd_manual.html" DESTINATION "${CMAKE_INSTALL_DOCDIR}")

From 8b674d7dc7f6570f3f56e6dff94e20dd7f1f26ff Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Fri, 17 Aug 2018 16:01:56 -0700
Subject: [PATCH 187/372] ensured compression level is maxed at
 ZSTD_maxCLevel()

---
 programs/fileio.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/programs/fileio.c b/programs/fileio.c
index 68b2f1593..02da1df95 100644
--- a/programs/fileio.c
+++ b/programs/fileio.c
@@ -860,10 +860,11 @@ FIO_compressZstdFrame(const cRess_t* ressPtr,
                             DISPLAYLEVEL(6, "slower speed , higher compression \n")
                             compressionLevel ++;
                             compressionLevel += (compressionLevel == 0);   /* skip 0 */
+                            if (compressionLevel > ZSTD_maxCLevel()) compressionLevel = ZSTD_maxCLevel();
                             ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_compressionLevel, (unsigned)compressionLevel);
                         }
                         if (speedChange == faster) {
-                            DISPLAYLEVEL(6, "slower speed , higher compression \n")
+                            DISPLAYLEVEL(6, "faster speed , lighter compression \n")
                             compressionLevel --;
                             compressionLevel -= (compressionLevel == 0);   /* skip 0 */
                             ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_compressionLevel, (unsigned)compressionLevel);

From 09e63c58ac38503b2338ac01229681048f0e3648 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Fri, 17 Aug 2018 16:20:27 -0700
Subject: [PATCH 188/372] fix : no longer slow down on input saturation

only slows down when all buffers are full
---
 programs/fileio.c | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/programs/fileio.c b/programs/fileio.c
index 02da1df95..fcb43030a 100644
--- a/programs/fileio.c
+++ b/programs/fileio.c
@@ -808,9 +808,12 @@ FIO_compressZstdFrame(const cRess_t* ressPtr,
 
                     cpszfp = zfp;
 
-                    if ( (zfp.ingested == cpszfp.ingested)
-                      && (zfp.consumed == cpszfp.consumed) ) {
-                        DISPLAYLEVEL(2, "no data read nor consumed : buffers are full (?) output is too slow => slow down ; or compression is slow + input has reached its limit => can't tell \n")
+                    if ( (zfp.ingested == cpszfp.ingested)   /* no data read : input buffer full */
+                      && (zfp.consumed == cpszfp.consumed)   /* no data compressed : no more buffer to compress OR compression is really slow */
+                      && (zfp.nbActiveWorkers == 0)          /* confirmed : no compression : either no more buffer to compress OR not enough data to start first worker */
+                      && (zfp.currentJobID > 0)              /* first job started : only remaining reason is no more available buffer to start compression */
+                      ) {
+                        DISPLAYLEVEL(6, "all buffers full : compression stopped => slow down \n")
                         speedChange = slower;
                     }
 

From 105677c6dbc5bfeaf8a0e52334104322d4618d5c Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Fri, 17 Aug 2018 18:11:54 -0700
Subject: [PATCH 189/372] created ZSTDMT_toFlushNow()

tells in a non-blocking way if there is something ready to flush right now.
only works with multi-threading for the time being.

Useful to know if flush speed will be limited by lack of production.
---
 lib/compress/zstd_compress.c   | 14 ++++++++++++++
 lib/compress/zstdmt_compress.c | 29 +++++++++++++++++++++++++++--
 lib/compress/zstdmt_compress.h | 22 ++++++++++++++++------
 lib/zstd.h                     | 11 ++++++++++-
 programs/fileio.c              | 11 ++++++++---
 5 files changed, 75 insertions(+), 12 deletions(-)

diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index 4bc18d2ec..34d4f5b0f 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -906,6 +906,20 @@ ZSTD_frameProgression ZSTD_getFrameProgression(const ZSTD_CCtx* cctx)
         return fp;
 }   }
 
+/*! ZSTD_toFlushNow()
+ *  Only useful for multithreading scenarios currently (nbWorkers >= 1).
+ */
+size_t ZSTD_toFlushNow(ZSTD_CCtx* cctx)
+{
+#ifdef ZSTD_MULTITHREAD
+    if (cctx->appliedParams.nbWorkers > 0) {
+        return ZSTDMT_toFlushNow(cctx->mtctx);
+    }
+#endif
+    return 0;   /* over-simplification; could also check if context is currently running in streaming mode, and in which case, report how many bytes are left to be flushed within output buffer */
+}
+
+
 
 static U32 ZSTD_equivalentCParams(ZSTD_compressionParameters cParams1,
                                   ZSTD_compressionParameters cParams2)
diff --git a/lib/compress/zstdmt_compress.c b/lib/compress/zstdmt_compress.c
index b61cd50ee..cc1af58e6 100644
--- a/lib/compress/zstdmt_compress.c
+++ b/lib/compress/zstdmt_compress.c
@@ -1096,20 +1096,45 @@ ZSTD_frameProgression ZSTDMT_getFrameProgression(ZSTDMT_CCtx* mtctx)
             ZSTD_pthread_mutex_lock(&jobPtr->job_mutex);
             {   size_t const cResult = jobPtr->cSize;
                 size_t const produced = ZSTD_isError(cResult) ? 0 : cResult;
+                size_t const flushed = ZSTD_isError(cResult) ? 0 : jobPtr->dstFlushed;
+                assert(flushed <= produced);
                 fps.ingested += jobPtr->src.size;
                 fps.consumed += jobPtr->consumed;
                 fps.produced += produced;
-                fps.flushed  += jobPtr->dstFlushed;
+                fps.flushed  += flushed;
                 fps.nbActiveWorkers += (jobPtr->consumed < jobPtr->src.size);
             }
             ZSTD_pthread_mutex_unlock(&mtctx->jobs[wJobID].job_mutex);
         }
     }
-    DEBUGLOG(5, "ZSTDMT_getFrameProgression : completed");
     return fps;
 }
 
 
+size_t ZSTDMT_toFlushNow(ZSTDMT_CCtx* mtctx)
+{
+    size_t toFlush;
+    unsigned const jobID = mtctx->doneJobID;
+    assert(jobID <= mtctx->nextJobID);
+    if (jobID == mtctx->nextJobID) return 0;   /* no active job => nothing to flush */
+
+    {   unsigned const wJobID = jobID & mtctx->jobIDMask;
+        ZSTDMT_jobDescription* jobPtr = &mtctx->jobs[wJobID];
+        ZSTD_pthread_mutex_lock(&jobPtr->job_mutex);
+        {   size_t const cResult = jobPtr->cSize;
+            size_t const produced = ZSTD_isError(cResult) ? 0 : cResult;
+            size_t const flushed = ZSTD_isError(cResult) ? 0 : jobPtr->dstFlushed;
+            assert(flushed <= produced);
+            toFlush = produced - flushed;
+            if (toFlush==0) assert(jobPtr->consumed < jobPtr->src.size);   /* if toFlush==0, doneJobID should still be active: if doneJobID is completed and fully flushed, ZSTDMT_flushProduced() should have already moved to next job */
+        }
+        ZSTD_pthread_mutex_unlock(&mtctx->jobs[wJobID].job_mutex);
+    }
+
+    return toFlush;
+}
+
+
 /* ------------------------------------------ */
 /* =====   Multi-threaded compression   ===== */
 /* ------------------------------------------ */
diff --git a/lib/compress/zstdmt_compress.h b/lib/compress/zstdmt_compress.h
index 34a475a42..12ad9f899 100644
--- a/lib/compress/zstdmt_compress.h
+++ b/lib/compress/zstdmt_compress.h
@@ -119,11 +119,21 @@ ZSTDLIB_API size_t ZSTDMT_compressStream_generic(ZSTDMT_CCtx* mtctx,
  * ===  Not exposed in libzstd. Never invoke directly   ===
  * ======================================================== */
 
+ /*! ZSTDMT_toFlushNow()
+  *  Tell how many bytes are ready to be flushed immediately.
+  *  Probe the oldest active job (not yet entirely flushed) and check its output buffer.
+  *  If return 0, it means there is no active job,
+  *  or, it means oldest job is still active, but everything produced has been flushed so far,
+  *  therefore flushing is limited by speed of oldest job. */
+size_t ZSTDMT_toFlushNow(ZSTDMT_CCtx* mtctx);
+
+/*! ZSTDMT_CCtxParam_setMTCtxParameter()
+ *  like ZSTDMT_setMTCtxParameter(), but into a ZSTD_CCtx_Params */
 size_t ZSTDMT_CCtxParam_setMTCtxParameter(ZSTD_CCtx_params* params, ZSTDMT_parameter parameter, unsigned value);
 
-/* ZSTDMT_CCtxParam_setNbWorkers()
- * Set nbWorkers, and clamp it.
- * Also reset jobSize and overlapLog */
+/*! ZSTDMT_CCtxParam_setNbWorkers()
+ *  Set nbWorkers, and clamp it.
+ *  Also reset jobSize and overlapLog */
 size_t ZSTDMT_CCtxParam_setNbWorkers(ZSTD_CCtx_params* params, unsigned nbWorkers);
 
 /*! ZSTDMT_updateCParams_whileCompressing() :
@@ -131,9 +141,9 @@ size_t ZSTDMT_CCtxParam_setNbWorkers(ZSTD_CCtx_params* params, unsigned nbWorker
  *  New parameters will be applied to next compression job. */
 void ZSTDMT_updateCParams_whileCompressing(ZSTDMT_CCtx* mtctx, const ZSTD_CCtx_params* cctxParams);
 
-/* ZSTDMT_getFrameProgression():
- * tells how much data has been consumed (input) and produced (output) for current frame.
- * able to count progression inside worker threads.
+/*! ZSTDMT_getFrameProgression():
+ *  tells how much data has been consumed (input) and produced (output) for current frame.
+ *  able to count progression inside worker threads.
  */
 ZSTD_frameProgression ZSTDMT_getFrameProgression(ZSTDMT_CCtx* mtctx);
 
diff --git a/lib/zstd.h b/lib/zstd.h
index 02e447b30..edb107c2b 100644
--- a/lib/zstd.h
+++ b/lib/zstd.h
@@ -746,7 +746,16 @@ typedef struct {
  * Therefore, (ingested - consumed) is amount of input data buffered internally, not yet compressed.
  * Can report progression inside worker threads (multi-threading and non-blocking mode).
  */
-ZSTD_frameProgression ZSTD_getFrameProgression(const ZSTD_CCtx* cctx);
+ZSTDLIB_API ZSTD_frameProgression ZSTD_getFrameProgression(const ZSTD_CCtx* cctx);
+
+/*! ZSTD_toFlushNow()
+ *  Tell how many bytes are ready to be flushed immediately.
+ *  Useful for multithreading scenarios (nbWorkers >= 1).
+ *  Probe the oldest active job (not yet entirely flushed) and check its output buffer.
+ *  If return 0, it means there is no active job, or
+ *  it means oldest job is still active, but everything produced has been flushed so far,
+ *  therefore flushing is limited by speed of oldest job. */
+ZSTDLIB_API size_t ZSTD_toFlushNow(ZSTD_CCtx* cctx);
 
 
 
diff --git a/programs/fileio.c b/programs/fileio.c
index fcb43030a..aeacd0440 100644
--- a/programs/fileio.c
+++ b/programs/fileio.c
@@ -747,6 +747,7 @@ FIO_compressZstdFrame(const cRess_t* ressPtr,
     /* stats */
     typedef enum { noChange, slower, faster } speedChange_e;
     speedChange_e speedChange = noChange;
+    unsigned flushWaiting = 0;
     unsigned inputPresented = 0;
     unsigned inputBlocked = 0;
     unsigned lastJobID = 0;
@@ -777,11 +778,13 @@ FIO_compressZstdFrame(const cRess_t* ressPtr,
 
             size_t const oldIPos = inBuff.pos;
             ZSTD_outBuffer outBuff = { ress.dstBuffer, ress.dstBufferSize, 0 };
+            size_t const toFlushNow = ZSTD_toFlushNow(ress.cctx);
             CHECK_V(stillToFlush, ZSTD_compress_generic(ress.cctx, &outBuff, &inBuff, directive));
 
             /* count stats */
             inputPresented++;
             if (oldIPos == inBuff.pos) inputBlocked++;
+            if (!toFlushNow) flushWaiting = 1;
 
             /* Write compressed stream */
             DISPLAYLEVEL(6, "ZSTD_compress_generic(end:%u) => input pos(%u)<=(%u)size ; output generated %u bytes \n",
@@ -817,11 +820,13 @@ FIO_compressZstdFrame(const cRess_t* ressPtr,
                         speedChange = slower;
                     }
 
-                    if ( (newlyProduced > (newlyFlushed * 9 / 8))
-                      && (stillToFlush > ZSTD_BLOCKSIZE_MAX) ) {
-                        DISPLAYLEVEL(6, "production faster than flushing (%llu > %llu) but there is still %u bytes to flush => slow down \n", newlyProduced, newlyFlushed, (U32)stillToFlush);
+                    if ( (newlyProduced > (newlyFlushed * 9 / 8))   /* compression produces more data than output can flush (though production can be spiky, due to work unit : (N==4)*block sizes) */
+                      && (flushWaiting == 0)                        /* flush speed was never slowed by lack of production, so it's operating at max capacity */
+                      ) {
+                        DISPLAYLEVEL(6, "compression faster than flush (%llu > %llu), and flushed was never slowed down by lack of production => slow down \n", newlyProduced, newlyFlushed);
                         speedChange = slower;
                     }
+                    flushWaiting = 0;
                 }
 
                 /* course correct only if there is at least one new job completed */

From c71c4f23d76f0668719d0e83b16ee572f5d9f0cb Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Mon, 20 Aug 2018 11:40:10 -0700
Subject: [PATCH 190/372] fix "unused parameter" in single-thread mode

within newly added ZSD_toFlushNow()
---
 lib/compress/zstd_compress.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index 34d4f5b0f..e09eef3d1 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -916,6 +916,7 @@ size_t ZSTD_toFlushNow(ZSTD_CCtx* cctx)
         return ZSTDMT_toFlushNow(cctx->mtctx);
     }
 #endif
+    (void)cctx;
     return 0;   /* over-simplification; could also check if context is currently running in streaming mode, and in which case, report how many bytes are left to be flushed within output buffer */
 }
 

From 78af534f82e32277d3272881d4351363ab1e3488 Mon Sep 17 00:00:00 2001
From: Eden Zik 
Date: Mon, 20 Aug 2018 22:15:24 -0400
Subject: [PATCH 191/372] Fixed unsafe string copy and concat in `fileio.c`.

Per warnings from flawfinder: "Does not check for buffer overflows when
copying to destination [MS-banned] (CWE-120). Consider using snprintf,
strcpy_s, or strlcpy (warning: strncpy easily misused).".

Replaced called to strcpy and strcat in `fileio.c` to calls with a
specified size (`strncpy` and `strncat`).

Tested the changes on OSX, Linux, Windows.
On OSX + Linux, changes were tested with ASAN. The following flags were
used: 'check_initialization_order=1:strict_init_order=1:detect_odr_violation=1:detect_stack_use_after_return=1'

To reproduce warning:
./flawfinder.py ./programs/fileio.c
---
 programs/fileio.c | 4 ++--
 tests/.gitignore  | 1 +
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/programs/fileio.c b/programs/fileio.c
index 39b2c741c..5f10958d7 100644
--- a/programs/fileio.c
+++ b/programs/fileio.c
@@ -1011,8 +1011,8 @@ int FIO_compressMultipleFilenames(const char** inFileNamesTable, unsigned nbFile
                 if (!dstFileName) {
                     EXM_THROW(30, "zstd: %s", strerror(errno));
             }   }
-            strcpy(dstFileName, inFileNamesTable[u]);
-            strcat(dstFileName, suffix);
+            strncpy(dstFileName, inFileNamesTable[u], ifnSize+1 /* Include null */);
+            strncat(dstFileName, suffix, suffixSize);
             missed_files += FIO_compressFilename_dstFile(ress, dstFileName, inFileNamesTable[u], compressionLevel);
     }   }
 
diff --git a/tests/.gitignore b/tests/.gitignore
index 4911b2d62..da536251d 100644
--- a/tests/.gitignore
+++ b/tests/.gitignore
@@ -26,6 +26,7 @@ invalidDictionaries
 checkTag
 zcat
 zstdcat
+tm
 
 # Tmp test directory
 zstdtest

From 77e805e3dbed9e14a2d50ad1c8965a79c39755aa Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Tue, 21 Aug 2018 18:19:27 -0700
Subject: [PATCH 192/372] bench: changed creation/reset function to
 timedFnState

for consistency
---
 programs/bench.c   | 148 ++++++++++++++--------------
 programs/bench.h   | 182 ++++++++++++++++++-----------------
 tests/paramgrill.c | 234 ++++++++++++++++++++++-----------------------
 3 files changed, 287 insertions(+), 277 deletions(-)

diff --git a/programs/bench.c b/programs/bench.c
index 79ef42caa..f280f00ef 100644
--- a/programs/bench.c
+++ b/programs/bench.c
@@ -124,8 +124,8 @@ static UTIL_time_t g_displayClock = UTIL_TIME_INITIALIZER;
 *  Benchmark Parameters
 ***************************************/
 
-BMK_advancedParams_t BMK_initAdvancedParams(void) { 
-    BMK_advancedParams_t res = { 
+BMK_advancedParams_t BMK_initAdvancedParams(void) {
+    BMK_advancedParams_t res = {
         BMK_both, /* mode */
         BMK_timeMode, /* loopMode */
         BMK_TIMETEST_DEFAULT_S, /* nbSeconds */
@@ -133,7 +133,7 @@ BMK_advancedParams_t BMK_initAdvancedParams(void) {
         0, /* nbWorkers */
         0, /* realTime */
         0, /* additionalParam */
-        0, /* ldmFlag */ 
+        0, /* ldmFlag */
         0, /* ldmMinMatch */
         0, /* ldmHashLog */
         0, /* ldmBuckSizeLog */
@@ -168,8 +168,8 @@ struct  BMK_timeState_t{
 #define MIN(a,b)    ((a) < (b) ? (a) : (b))
 #define MAX(a,b)    ((a) > (b) ? (a) : (b))
 
-static void BMK_initCCtx(ZSTD_CCtx* ctx, 
-    const void* dictBuffer, size_t dictBufferSize, int cLevel, 
+static void BMK_initCCtx(ZSTD_CCtx* ctx,
+    const void* dictBuffer, size_t dictBufferSize, int cLevel,
     const ZSTD_compressionParameters* comprParams, const BMK_advancedParams_t* adv) {
     ZSTD_CCtx_reset(ctx);
     ZSTD_CCtx_resetParameters(ctx);
@@ -195,7 +195,7 @@ static void BMK_initCCtx(ZSTD_CCtx* ctx,
 }
 
 
-static void BMK_initDCtx(ZSTD_DCtx* dctx, 
+static void BMK_initDCtx(ZSTD_DCtx* dctx,
     const void* dictBuffer, size_t dictBufferSize) {
     ZSTD_DCtx_reset(dctx);
     ZSTD_DCtx_loadDictionary(dctx, dictBuffer, dictBufferSize);
@@ -230,8 +230,8 @@ static size_t local_initDCtx(void* payload) {
 
 /* additional argument is just the context */
 static size_t local_defaultCompress(
-    const void* srcBuffer, size_t srcSize, 
-    void* dstBuffer, size_t dstSize, 
+    const void* srcBuffer, size_t srcSize,
+    void* dstBuffer, size_t dstSize,
     void* addArgs) {
     size_t moreToFlush = 1;
     ZSTD_CCtx* ctx = (ZSTD_CCtx*)addArgs;
@@ -257,8 +257,8 @@ static size_t local_defaultCompress(
 
 /* additional argument is just the context */
 static size_t local_defaultDecompress(
-    const void* srcBuffer, size_t srcSize, 
-    void* dstBuffer, size_t dstSize, 
+    const void* srcBuffer, size_t srcSize,
+    void* dstBuffer, size_t dstSize,
     void* addArgs) {
     size_t moreToFlush = 1;
     ZSTD_DCtx* dctx = (ZSTD_DCtx*)addArgs;
@@ -294,7 +294,7 @@ BMK_customReturn_t BMK_benchFunction(
     BMK_initFn_t initFn, void* initPayload,
     size_t blockCount,
     const void* const * const srcBlockBuffers, const size_t* srcBlockSizes,
-    void* const * const dstBlockBuffers, const size_t* dstBlockCapacities, size_t* blockResult, 
+    void* const * const dstBlockBuffers, const size_t* dstBlockCapacities, size_t* blockResult,
     unsigned nbLoops) {
     size_t dstSize = 0;
     U64 totalTime;
@@ -312,15 +312,15 @@ BMK_customReturn_t BMK_benchFunction(
             memset(dstBlockBuffers[i], 0xE5, dstBlockCapacities[i]);  /* warm up and erase result buffer */
         }
 #if 0
-        /* based on testing these seem to lower accuracy of multiple calls of 1 nbLoops vs 1 call of multiple nbLoops 
-         * (Makes former slower)   
+        /* based on testing these seem to lower accuracy of multiple calls of 1 nbLoops vs 1 call of multiple nbLoops
+         * (Makes former slower)
          */
         UTIL_sleepMilli(5);  /* give processor time to other processes */
         UTIL_waitForNextTick();
 #endif
     }
 
-    {      
+    {
         unsigned i, j;
         clockStart = UTIL_getTime();
         if(initFn != NULL) { initFn(initPayload); }
@@ -335,7 +335,7 @@ BMK_customReturn_t BMK_benchFunction(
                     if(blockResult != NULL) {
                         blockResult[j] = res;
                     }
-                } 
+                }
             }
         }
         totalTime = UTIL_clockSpanNano(clockStart);
@@ -345,27 +345,27 @@ BMK_customReturn_t BMK_benchFunction(
     retval.result.nanoSecPerRun = totalTime / nbLoops;
     retval.result.sumOfReturn = dstSize;
     return retval;
-} 
+}
 
 #define MINUSABLETIME 500000000ULL /* 0.5 seconds in ns */
 
-void BMK_resetTimeState(BMK_timedFnState_t* r, unsigned nbSeconds) {
+void BMK_resetTimedFnState(BMK_timedFnState_t* r, unsigned nbSeconds) {
     r->nbLoops = 1;
     r->timeRemaining = (U64)nbSeconds * TIMELOOP_NANOSEC;
     r->coolTime = UTIL_getTime();
     r->fastestTime = (U64)(-1LL);
 }
 
-BMK_timedFnState_t* BMK_createTimeState(unsigned nbSeconds) {
+BMK_timedFnState_t* BMK_createTimedFnState(unsigned nbSeconds) {
     BMK_timedFnState_t* r = (BMK_timedFnState_t*)malloc(sizeof(struct BMK_timeState_t));
     if(r == NULL) {
         return r;
     }
-    BMK_resetTimeState(r, nbSeconds);
+    BMK_resetTimedFnState(r, nbSeconds);
     return r;
 }
 
-void BMK_freeTimeState(BMK_timedFnState_t* state) {
+void BMK_freeTimedFnState(BMK_timedFnState_t* state) {
     free(state);
 }
 
@@ -376,7 +376,7 @@ BMK_customTimedReturn_t BMK_benchFunctionTimed(
     BMK_initFn_t initFn, void* initPayload,
     size_t blockCount,
     const void* const* const srcBlockBuffers, const size_t* srcBlockSizes,
-    void * const * const dstBlockBuffers, const size_t * dstBlockCapacities, size_t* blockResults) 
+    void * const * const dstBlockBuffers, const size_t * dstBlockCapacities, size_t* blockResults)
 {
     U64 fastest = cont->fastestTime;
     int completed = 0;
@@ -402,7 +402,7 @@ BMK_customTimedReturn_t BMK_benchFunctionTimed(
         {   U64 const loopDuration = r.result.result.nanoSecPerRun * cont->nbLoops;
             r.completed = (cont->timeRemaining <= loopDuration);
             cont->timeRemaining -= loopDuration;
-            if (loopDuration > (TIMELOOP_NANOSEC / 100)) { 
+            if (loopDuration > (TIMELOOP_NANOSEC / 100)) {
                 fastest = MIN(fastest, r.result.result.nanoSecPerRun);
                 if(loopDuration >= MINUSABLETIME) {
                     r.result.result.nanoSecPerRun = fastest;
@@ -427,7 +427,7 @@ BMK_customTimedReturn_t BMK_benchFunctionTimed(
 /* benchMem with no allocation */
 static BMK_return_t BMK_benchMemAdvancedNoAlloc(
     const void ** const srcPtrs, size_t* const srcSizes,
-    void** const cPtrs, size_t* const cCapacities, size_t* const cSizes, 
+    void** const cPtrs, size_t* const cCapacities, size_t* const cSizes,
     void** const resPtrs, size_t* const resSizes,
     void** resultBufferPtr, void* compressedBuffer,
     const size_t maxCompressedSize,
@@ -438,7 +438,7 @@ static BMK_return_t BMK_benchMemAdvancedNoAlloc(
     const int cLevel, const ZSTD_compressionParameters* comprParams,
     const void* dictBuffer, size_t dictBufferSize,
     ZSTD_CCtx* ctx, ZSTD_DCtx* dctx,
-    int displayLevel, const char* displayName, const BMK_advancedParams_t* adv) 
+    int displayLevel, const char* displayName, const BMK_advancedParams_t* adv)
 {
     size_t const blockSize = ((adv->blockSize>=32 && (adv->mode != BMK_decodeOnly)) ? adv->blockSize : srcSize) + (!srcSize); /* avoid div by 0 */
     BMK_return_t results = { { 0, 0, 0, 0 }, 0 } ;
@@ -447,7 +447,7 @@ static BMK_return_t BMK_benchMemAdvancedNoAlloc(
     double ratio = 0.;
     U32 nbBlocks;
 
-    if(!ctx || !dctx) 
+    if(!ctx || !dctx)
         EXM_THROW(31, BMK_return_t, "error: passed in null context");
 
     /* init */
@@ -466,16 +466,16 @@ static BMK_return_t BMK_benchMemAdvancedNoAlloc(
             free(*resultBufferPtr);
             *resultBufferPtr = malloc(decodedSize);
             if (!(*resultBufferPtr)) {
-                EXM_THROW(33, BMK_return_t, "not enough memory"); 
+                EXM_THROW(33, BMK_return_t, "not enough memory");
             }
             if (totalDSize64 > decodedSize) {
-                free(*resultBufferPtr); 
+                free(*resultBufferPtr);
                 EXM_THROW(32, BMK_return_t, "original size is too large");   /* size_t overflow */
             }
             cSize = srcSize;
             srcSize = decodedSize;
             ratio = (double)srcSize / (double)cSize;
-        }   
+        }
     }
 
     /* Init data blocks  */
@@ -499,8 +499,8 @@ static BMK_return_t BMK_benchMemAdvancedNoAlloc(
                 cPtr += cCapacities[nbBlocks];
                 resPtr += thisBlockSize;
                 remaining -= thisBlockSize;
-            }   
-        }   
+            }
+        }
     }
 
     /* warmimg up memory */
@@ -511,7 +511,7 @@ static BMK_return_t BMK_benchMemAdvancedNoAlloc(
     }
 
     /* Bench */
-    {   
+    {
         U64 const crcOrig = (adv->mode == BMK_decodeOnly) ? 0 : XXH64(srcBuffer, srcSize, 0);
 #       define NB_MARKS 4
         const char* const marks[NB_MARKS] = { " |", " /", " =",  "\\" };
@@ -545,7 +545,7 @@ static BMK_return_t BMK_benchMemAdvancedNoAlloc(
                     intermediateResultDecompress.completed = 0;
                 }
                 while(!(intermediateResultCompress.completed && intermediateResultDecompress.completed)) {
-                    if(!intermediateResultCompress.completed) { 
+                    if(!intermediateResultCompress.completed) {
                         intermediateResultCompress = BMK_benchFunctionTimed(timeStateCompress, &local_defaultCompress, (void*)ctx, &local_initCCtx, (void*)&cctxprep,
                         nbBlocks, srcPtrs, srcSizes, cPtrs, cCapacities, cSizes);
                         if(intermediateResultCompress.result.error) {
@@ -553,7 +553,7 @@ static BMK_return_t BMK_benchMemAdvancedNoAlloc(
                             return results;
                         }
                         ratio = (double)(srcSize / intermediateResultCompress.result.result.sumOfReturn);
-                        {   
+                        {
                             int const ratioAccuracy = (ratio < 10.) ? 3 : 2;
                             results.result.cSpeed = (srcSize * TIMELOOP_NANOSEC / intermediateResultCompress.result.result.nanoSecPerRun);
                             cSize = intermediateResultCompress.result.result.sumOfReturn;
@@ -574,8 +574,8 @@ static BMK_return_t BMK_benchMemAdvancedNoAlloc(
                             results.error = intermediateResultDecompress.result.error;
                             return results;
                         }
-      
-                        {   
+
+                        {
                             int const ratioAccuracy = (ratio < 10.) ? 3 : 2;
                             results.result.dSpeed = (srcSize * TIMELOOP_NANOSEC/ intermediateResultDecompress.result.result.nanoSecPerRun);
                             markNb = (markNb+1) % NB_MARKS;
@@ -592,7 +592,7 @@ static BMK_return_t BMK_benchMemAdvancedNoAlloc(
                 if(adv->mode != BMK_decodeOnly) {
 
                     BMK_customReturn_t compressionResults = BMK_benchFunction(&local_defaultCompress, (void*)ctx, &local_initCCtx, (void*)&cctxprep,
-                        nbBlocks, srcPtrs, srcSizes, cPtrs, cCapacities, cSizes, adv->nbSeconds); 
+                        nbBlocks, srcPtrs, srcSizes, cPtrs, cCapacities, cSizes, adv->nbSeconds);
                     if(compressionResults.error) {
                         results.error = compressionResults.error;
                         return results;
@@ -605,7 +605,7 @@ static BMK_return_t BMK_benchMemAdvancedNoAlloc(
                     }
 
                     results.result.cSize = compressionResults.result.sumOfReturn;
-                    {   
+                    {
                         int const ratioAccuracy = (ratio < 10.) ? 3 : 2;
                         cSize = compressionResults.result.sumOfReturn;
                         results.result.cSize = cSize;
@@ -621,7 +621,7 @@ static BMK_return_t BMK_benchMemAdvancedNoAlloc(
                     BMK_customReturn_t decompressionResults = BMK_benchFunction(
                         &local_defaultDecompress, (void*)(dctx),
                         &local_initDCtx, (void*)&dctxprep, nbBlocks,
-                        (const void* const*)cPtrs, cSizes, resPtrs, resSizes, NULL, 
+                        (const void* const*)cPtrs, cSizes, resPtrs, resSizes, NULL,
                         adv->nbSeconds);
                     if(decompressionResults.error) {
                         results.error = decompressionResults.error;
@@ -634,7 +634,7 @@ static BMK_return_t BMK_benchMemAdvancedNoAlloc(
                         results.result.dSpeed = srcSize * TIMELOOP_NANOSEC / decompressionResults.result.nanoSecPerRun;
                     }
 
-                    {   
+                    {
                         int const ratioAccuracy = (ratio < 10.) ? 3 : 2;
                         markNb = (markNb+1) % NB_MARKS;
                         DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.*f),%6.*f MB/s ,%6.1f MB/s \r",
@@ -683,9 +683,9 @@ static BMK_return_t BMK_benchMemAdvancedNoAlloc(
                     }
                     if (u==srcSize-1) {  /* should never happen */
                         DISPLAY("no difference detected\n");
-                    }   
+                    }
                 }
-            }   
+            }
         }   /* CRC Checking */
 
     if (displayLevel == 1) {   /* hidden display mode -q, used by python speed benchmark */
@@ -705,7 +705,7 @@ static BMK_return_t BMK_benchMemAdvancedNoAlloc(
 }
 
 BMK_return_t BMK_benchMemAdvanced(const void* srcBuffer, size_t srcSize,
-                        void* dstBuffer, size_t dstCapacity, 
+                        void* dstBuffer, size_t dstCapacity,
                         const size_t* fileSizes, unsigned nbFiles,
                         const int cLevel, const ZSTD_compressionParameters* comprParams,
                         const void* dictBuffer, size_t dictBufferSize,
@@ -727,8 +727,8 @@ BMK_return_t BMK_benchMemAdvanced(const void* srcBuffer, size_t srcSize,
     void ** const resPtrs = (void**)malloc(maxNbBlocks * sizeof(void*));
     size_t* const resSizes = (size_t*)malloc(maxNbBlocks * sizeof(size_t));
 
-    BMK_timedFnState_t* timeStateCompress = BMK_createTimeState(adv->nbSeconds);
-    BMK_timedFnState_t* timeStateDecompress = BMK_createTimeState(adv->nbSeconds);
+    BMK_timedFnState_t* timeStateCompress = BMK_createTimedFnState(adv->nbSeconds);
+    BMK_timedFnState_t* timeStateDecompress = BMK_createTimedFnState(adv->nbSeconds);
 
     ZSTD_CCtx* ctx = ZSTD_createCCtx();
     ZSTD_DCtx* dctx = ZSTD_createDCtx();
@@ -744,8 +744,8 @@ BMK_return_t BMK_benchMemAdvanced(const void* srcBuffer, size_t srcSize,
 
     void* resultBuffer = srcSize ? malloc(srcSize) : NULL;
 
-    int allocationincomplete = !srcPtrs || !srcSizes || !cPtrs || 
-        !cSizes || !cCapacities || !resPtrs || !resSizes || 
+    int allocationincomplete = !srcPtrs || !srcSizes || !cPtrs ||
+        !cSizes || !cCapacities || !resPtrs || !resSizes ||
         !timeStateCompress || !timeStateDecompress || !compressedBuffer || !resultBuffer;
 
 
@@ -756,20 +756,20 @@ BMK_return_t BMK_benchMemAdvanced(const void* srcBuffer, size_t srcSize,
             srcBuffer, srcSize, fileSizes, nbFiles, cLevel, comprParams,
             dictBuffer, dictBufferSize, ctx, dctx, displayLevel, displayName, adv);
     }
-    
+
     /* clean up */
-    BMK_freeTimeState(timeStateCompress);
-    BMK_freeTimeState(timeStateDecompress);
-    
+    BMK_freeTimedFnState(timeStateCompress);
+    BMK_freeTimedFnState(timeStateDecompress);
+
     ZSTD_freeCCtx(ctx);
     ZSTD_freeDCtx(dctx);
 
     free(internalDstBuffer);
     free(resultBuffer);
 
-    free((void*)srcPtrs); 
-    free(srcSizes); 
-    free(cPtrs); 
+    free((void*)srcPtrs);
+    free(srcSizes);
+    free(cPtrs);
     free(cSizes);
     free(cCapacities);
     free(resPtrs);
@@ -778,9 +778,9 @@ BMK_return_t BMK_benchMemAdvanced(const void* srcBuffer, size_t srcSize,
     if(allocationincomplete) {
         EXM_THROW(31, BMK_return_t, "allocation error : not enough memory");
     }
-    
+
     if(parametersConflict) {
-        EXM_THROW(32, BMK_return_t, "Conflicting input results");   
+        EXM_THROW(32, BMK_return_t, "Conflicting input results");
     }
     return results;
 }
@@ -840,11 +840,11 @@ static BMK_return_t BMK_benchCLevel(const void* srcBuffer, size_t benchedSize,
     if (displayLevel == 1 && !adv->additionalParam)
         DISPLAY("bench %s %s: input %u bytes, %u seconds, %u KB blocks\n", ZSTD_VERSION_STRING, ZSTD_GIT_COMMIT_STRING, (U32)benchedSize, adv->nbSeconds, (U32)(adv->blockSize>>10));
 
-    res = BMK_benchMemAdvanced(srcBuffer, benchedSize, 
-                NULL, 0, 
-                fileSizes, nbFiles, 
-                cLevel, comprParams, 
-                dictBuffer, dictBufferSize, 
+    res = BMK_benchMemAdvanced(srcBuffer, benchedSize,
+                NULL, 0,
+                fileSizes, nbFiles,
+                cLevel, comprParams,
+                dictBuffer, dictBufferSize,
                 displayLevel, displayName, adv);
 
     return res;
@@ -855,7 +855,7 @@ static BMK_return_t BMK_benchCLevel(const void* srcBuffer, size_t benchedSize,
  *  Loads `buffer` with content of files listed within `fileNamesTable`.
  *  At most, fills `buffer` entirely. */
 static int BMK_loadFiles(void* buffer, size_t bufferSize,
-                          size_t* fileSizes, const char* const * const fileNamesTable, 
+                          size_t* fileSizes, const char* const * const fileNamesTable,
                           unsigned nbFiles, int displayLevel)
 {
     size_t pos = 0, totalSize = 0;
@@ -890,8 +890,8 @@ static int BMK_loadFiles(void* buffer, size_t bufferSize,
 }
 
 BMK_return_t BMK_benchFilesAdvanced(const char* const * const fileNamesTable, unsigned const nbFiles,
-                               const char* const dictFileName, int const cLevel, 
-                               const ZSTD_compressionParameters* const compressionParams, 
+                               const char* const dictFileName, int const cLevel,
+                               const ZSTD_compressionParameters* const compressionParams,
                                int displayLevel, const BMK_advancedParams_t * const adv)
 {
     void* srcBuffer = NULL;
@@ -960,13 +960,13 @@ BMK_return_t BMK_benchFilesAdvanced(const char* const * const fileNamesTable, un
     {
         char mfName[20] = {0};
         snprintf (mfName, sizeof(mfName), " %u files", nbFiles);
-        {   
+        {
             const char* const displayName = (nbFiles > 1) ? mfName : fileNamesTable[0];
-            res = BMK_benchCLevel(srcBuffer, benchedSize, 
-                            fileSizes, nbFiles, 
+            res = BMK_benchCLevel(srcBuffer, benchedSize,
+                            fileSizes, nbFiles,
                             cLevel, compressionParams,
-                            dictBuffer, dictBufferSize, 
-                            displayLevel, displayName, 
+                            dictBuffer, dictBufferSize,
+                            displayLevel, displayName,
                             adv);
     }   }
 
@@ -1000,10 +1000,10 @@ BMK_return_t BMK_syntheticTest(int cLevel, double compressibility,
 
     /* Bench */
     snprintf (name, sizeof(name), "Synthetic %2u%%", (unsigned)(compressibility*100));
-    res = BMK_benchCLevel(srcBuffer, benchedSize, 
-                    &benchedSize, 1, 
-                    cLevel, compressionParams, 
-                    NULL, 0, 
+    res = BMK_benchCLevel(srcBuffer, benchedSize,
+                    &benchedSize, 1,
+                    cLevel, compressionParams,
+                    NULL, 0,
                     displayLevel, name, adv);
 
     /* clean up */
@@ -1013,8 +1013,8 @@ BMK_return_t BMK_syntheticTest(int cLevel, double compressibility,
 }
 
 BMK_return_t BMK_benchFiles(const char* const * const fileNamesTable, unsigned const nbFiles,
-                   const char* const dictFileName, 
-                   int const cLevel, const ZSTD_compressionParameters* const compressionParams, 
+                   const char* const dictFileName,
+                   int const cLevel, const ZSTD_compressionParameters* const compressionParams,
                    int displayLevel) {
     const BMK_advancedParams_t adv = BMK_initAdvancedParams();
     return BMK_benchFilesAdvanced(fileNamesTable, nbFiles, dictFileName, cLevel, compressionParams, displayLevel, &adv);
diff --git a/programs/bench.h b/programs/bench.h
index ad4ede1f0..ec3c5ef0b 100644
--- a/programs/bench.h
+++ b/programs/bench.h
@@ -19,11 +19,12 @@ extern "C" {
 #define ZSTD_STATIC_LINKING_ONLY   /* ZSTD_compressionParameters */
 #include "zstd.h"     /* ZSTD_compressionParameters */
 
-/* Creates a struct of type typeName with an int type .error field
- * and a .result field of some baseType. Functions with return
- * typeName pass a successful result with .error = 0 and .result
- * with the intended result, while returning an error will result
- * in .error != 0. 
+/* Creates a struct type typeName, featuring:
+ * - an .error field of type int
+ * - a .result field of some baseType.
+ * Functions with return type typeName
+ * will either be successful, with .error = 0, providing a valid .result,
+ * or return an error, with .error != 0, in which case .result is invalid.
  */
 #define ERROR_STRUCT(baseType, typeName) typedef struct { \
     baseType result; \
@@ -32,36 +33,37 @@ extern "C" {
 
 typedef struct {
     size_t cSize;
-    U64 cSpeed;   /* bytes / sec */
-    U64 dSpeed;
-    size_t cMem;
+    unsigned long long cSpeed;   /* bytes / sec */
+    unsigned long long dSpeed;
+    size_t cMem;                 /* ? what does it reports ? */
 } BMK_result_t;
 
 ERROR_STRUCT(BMK_result_t, BMK_return_t);
 
-/* called in cli */
-/* Loads files in fileNamesTable into memory, as well as a dictionary 
- * from dictFileName, and then uses benchMem */
-/* fileNamesTable - name of files to benchmark
- * nbFiles - number of files (size of fileNamesTable), must be > 0
- * dictFileName - name of dictionary file to load
- * cLevel - compression level to benchmark, errors if invalid
- * compressionParams - basic compression Parameters
- * displayLevel - what gets printed
- *      0 : no display;   
- *      1 : errors;   
- *      2 : + result + interaction + warnings;   
- *      3 : + progression;   
+/*! BMK_benchFiles() -- called by zstdcli */
+/*  Loads files from fileNamesTable into memory,
+ *  loads dictionary from dictFileName,
+ *  then uses benchMem().
+ *  fileNamesTable - name of files to benchmark
+ *  nbFiles - number of files (size of fileNamesTable), must be > 0  (what happens if not ?)
+ *  dictFileName - name of dictionary file to load
+ *  cLevel - compression level to benchmark, errors if invalid
+ *  compressionParams - advanced compression Parameters
+ *  displayLevel - what gets printed
+ *      0 : no display;
+ *      1 : errors;
+ *      2 : + result + interaction + warnings;
+ *      3 : + progression;
  *      4 : + information
- * return 
+ * @return
  *      .error will give a nonzero error value if an error has occured
- *      .result - if .error = 0, .result will return the time taken to compression speed
- *          (.cSpeed), decompression speed (.dSpeed), and compressed size (.cSize) of the original
- *          file
+ *      .result - only valid if .error = 0,
+ *          .result will return compression speed (.cSpeed),
+ *          decompression speed (.dSpeed), and compressed size (.cSize).
  */
-BMK_return_t BMK_benchFiles(const char* const * const fileNamesTable, unsigned const nbFiles,
-                   const char* const dictFileName, 
-                   int const cLevel, const ZSTD_compressionParameters* const compressionParams, 
+BMK_return_t BMK_benchFiles(const char* const * fileNamesTable, unsigned nbFiles,
+                   const char* dictFileName,
+                   int cLevel, const ZSTD_compressionParameters* compressionParams,
                    int displayLevel);
 
 typedef enum {
@@ -85,7 +87,7 @@ typedef struct {
     int additionalParam;        /* used by python speed benchmark */
     unsigned ldmFlag;           /* enables long distance matching */
     unsigned ldmMinMatch;       /* below: parameters for long distance matching, see zstd.1.md for meaning */
-    unsigned ldmHashLog; 
+    unsigned ldmHashLog;
     unsigned ldmBucketSizeLog;
     unsigned ldmHashEveryLog;
 } BMK_advancedParams_t;
@@ -93,67 +95,75 @@ typedef struct {
 /* returns default parameters used by nonAdvanced functions */
 BMK_advancedParams_t BMK_initAdvancedParams(void);
 
-/* See benchFiles for normal parameter uses and return, see advancedParams_t for adv */
-BMK_return_t BMK_benchFilesAdvanced(const char* const * const fileNamesTable, unsigned const nbFiles,
-                   const char* const dictFileName, 
-                   int const cLevel, const ZSTD_compressionParameters* const compressionParams, 
-                   int displayLevel, const BMK_advancedParams_t* const adv);
+/*! BMK_benchFilesAdvanced():
+ *  Same as BMK_benchFiles(),
+ *  with more controls, provided through advancedParams_t structure */
+BMK_return_t BMK_benchFilesAdvanced(const char* const * fileNamesTable, unsigned nbFiles,
+                   const char* dictFileName,
+                   int cLevel, const ZSTD_compressionParameters* compressionParams,
+                   int displayLevel, const BMK_advancedParams_t* adv);
 
-/* called in cli */
-/* Generates a sample with datagen with the compressibility argument*/
-/* cLevel - compression level to benchmark, errors if invalid
- * compressibility - determines compressibility of sample
- * compressionParams - basic compression Parameters
- * displayLevel - see benchFiles
- * adv - see advanced_Params_t
- * return 
+/*! BMK_syntheticTest() -- called from zstdcli */
+/*  Generates a sample with datagen, using compressibility argument */
+/*  cLevel - compression level to benchmark, errors if invalid
+ *  compressibility - determines compressibility of sample
+ *  compressionParams - basic compression Parameters
+ *  displayLevel - see benchFiles
+ *  adv - see advanced_Params_t
+ * @return:
  *      .error will give a nonzero error value if an error has occured
- *      .result - if .error = 0, .result will return the time taken to compression speed
- *          (.cSpeed), decompression speed (.dSpeed), and compressed size (.cSize) of the original
- *          file
+ *      .result - only valid if .error = 0,
+ *          .result will return the compression speed (.cSpeed),
+ *          decompression speed (.dSpeed), and compressed size (.cSize).
  */
 BMK_return_t BMK_syntheticTest(int cLevel, double compressibility,
                               const ZSTD_compressionParameters* compressionParams,
                               int displayLevel, const BMK_advancedParams_t * const adv);
 
-/* basic benchmarking function, called in paramgrill 
- * applies ZSTD_compress_generic() and ZSTD_decompress_generic() on data in srcBuffer
- * with specific compression parameters specified by other arguments using benchFunction
- * (cLevel, comprParams + adv in advanced Mode) */
-/* srcBuffer - data source, expected to be valid compressed data if in Decode Only Mode
- * srcSize - size of data in srcBuffer
- * cLevel - compression level  
- * comprParams - basic compression parameters
- * dictBuffer - a dictionary if used, null otherwise
- * dictBufferSize - size of dictBuffer, 0 otherwise
- * diplayLevel - see BMK_benchFiles
- * displayName - name used by display
- * return
+/** BMK_benchMem() -- core benchmarking function, called in paramgrill
+ *  applies ZSTD_compress_generic() and ZSTD_decompress_generic() on data in srcBuffer
+ *  with specific compression parameters provided by other arguments using benchFunction
+ *  (cLevel, comprParams + adv in advanced Mode) */
+/*  srcBuffer - data source, expected to be valid compressed data if in Decode Only Mode
+ *  srcSize - size of data in srcBuffer
+ *  fileSizes - srcBuffer is considered cut into 1+ segments, to compress separately.
+ *              note : sum(fileSizes) must be == srcSize.  (<== ensure it's properly checked)
+ *  nbFiles - nb of segments
+ *  cLevel - compression level
+ *  comprParams - basic compression parameters
+ *  dictBuffer - a dictionary if used, null otherwise
+ *  dictBufferSize - size of dictBuffer, 0 otherwise
+ *  diplayLevel - see BMK_benchFiles
+ *  displayName - name used by display
+ * @return
  *      .error will give a nonzero value if an error has occured
- *      .result - if .error = 0, will give the same results as benchFiles
+ *      .result - only valid if .error = 0,
+ *          provide the same results as benchFiles()
  *          but for the data stored in srcBuffer
  */
 BMK_return_t BMK_benchMem(const void* srcBuffer, size_t srcSize,
                         const size_t* fileSizes, unsigned nbFiles,
-                        const int cLevel, const ZSTD_compressionParameters* comprParams,
+                        int cLevel, const ZSTD_compressionParameters* comprParams,
                         const void* dictBuffer, size_t dictBufferSize,
                         int displayLevel, const char* displayName);
 
-/* See benchMem for normal parameter uses and return, see advancedParams_t for adv 
+/* BMK_benchMemAdvanced() : same as BMK_benchMem()
+ * with following additional options :
  * dstBuffer - destination buffer to write compressed output in, NULL if none provided.
  * dstCapacity - capacity of destination buffer, give 0 if dstBuffer = NULL
+ * adv = see advancedParams_t
  */
 BMK_return_t BMK_benchMemAdvanced(const void* srcBuffer, size_t srcSize,
-                        void* dstBuffer, size_t dstCapacity, 
+                        void* dstBuffer, size_t dstCapacity,
                         const size_t* fileSizes, unsigned nbFiles,
-                        const int cLevel, const ZSTD_compressionParameters* comprParams,
+                        int cLevel, const ZSTD_compressionParameters* comprParams,
                         const void* dictBuffer, size_t dictBufferSize,
                         int displayLevel, const char* displayName,
                         const BMK_advancedParams_t* adv);
 
 typedef struct {
     size_t sumOfReturn;    /* sum of return values */
-    U64 nanoSecPerRun;     /* time per iteration */
+    unsigned long long nanoSecPerRun;     /* time per iteration */
 } BMK_customResult_t;
 
 ERROR_STRUCT(BMK_customResult_t, BMK_customReturn_t);
@@ -165,60 +175,60 @@ typedef size_t (*BMK_initFn_t)(void*);
 
 /* benchFn - (*benchFn)(srcBuffers[i], srcSizes[i], dstBuffers[i], dstCapacities[i], benchPayload)
  *      is run nbLoops times
- * initFn - (*initFn)(initPayload) is run once per benchmark at the beginning. This argument can 
+ * initFn - (*initFn)(initPayload) is run once per benchmark at the beginning. This argument can
  *          be NULL, in which case nothing is run.
- * blockCount - number of blocks (size of srcBuffers, srcSizes, dstBuffers, dstCapacities)
+ * blockCount - number of blocks. Size of all array parameters : srcBuffers, srcSizes, dstBuffers, dstCapacities, blockResults
  * srcBuffers - an array of buffers to be operated on by benchFn
  * srcSizes - an array of the sizes of above buffers
  * dstBuffers - an array of buffers to be written into by benchFn
  * dstCapacities - an array of the capacities of above buffers
  * blockResults - the return value of benchFn called on each block.
  * nbLoops - defines number of times benchFn is run.
- * assumed array of size blockCount, will have compressed size of each block written to it.
- * return 
+ * @return:
  *      .error will give a nonzero value if ZSTD_isError() is nonzero for any of the return
  *          of the calls to initFn and benchFn, or if benchFunction errors internally
- *      .result - if .error = 0, then .result will contain the sum of all return values of 
- *          benchFn on the first iteration through all of the blocks (.sumOfReturn) and also 
- *          the time per run of benchFn (.nanoSecPerRun). For the former, this
- *          is generally intended to be used on functions which return the # of bytes written 
- *          into dstBuffer, hence this value will be the total amount of bytes written to 
- *          dstBuffer.
+ *      .result - if .error = 0, then .result will contain
+ *          the sum of all return values of benchFn on the first iteration through all of the blocks (.sumOfReturn)
+ *          and also the time per run of benchFn (.nanoSecPerRun).
+ *          For the former, this is generally intended to be used on functions which return the # of bytes written into dstBuffer,
+ *          hence this value will be the total amount of bytes written into dstBuffer.
  */
 BMK_customReturn_t BMK_benchFunction(BMK_benchFn_t benchFn, void* benchPayload,
                         BMK_initFn_t initFn, void* initPayload,
                         size_t blockCount,
-                        const void* const * const srcBuffers, const size_t* srcSizes,
-                        void * const * const dstBuffers, const size_t* dstCapacities, size_t* blockResults,  
+                        const void *const * srcBuffers, const size_t* srcSizes,
+                        void *const * dstBuffers, const size_t* dstCapacities, size_t* blockResults,
                         unsigned nbLoops);
 
 
 /* state information needed to advance computation for benchFunctionTimed */
 typedef struct BMK_timeState_t BMK_timedFnState_t;
 /* initializes timeState object with desired number of seconds */
-BMK_timedFnState_t* BMK_createTimeState(unsigned nbSeconds);
+BMK_timedFnState_t* BMK_createTimedFnState(unsigned nbSeconds);
 /* resets existing timeState object */
-void BMK_resetTimeState(BMK_timedFnState_t*, unsigned nbSeconds);
+void BMK_resetTimedFnState(BMK_timedFnState_t*, unsigned nbSeconds);
 /* deletes timeState object */
-void BMK_freeTimeState(BMK_timedFnState_t* state);
+void BMK_freeTimedFnState(BMK_timedFnState_t* state);
 
 typedef struct {
     BMK_customReturn_t result;
     int completed;
 } BMK_customTimedReturn_t;
 
-/* 
- * Benchmarks custom functions like BMK_benchFunction(), but runs for nbSeconds seconds rather than a fixed number of loops
- * arguments mostly the same other than BMK_benchFunction()
- * Usage - benchFunctionTimed will return in approximately one second. Keep calling BMK_benchFunctionTimed() until the return's completed field = 1. 
- * to continue updating intermediate result. Intermediate return values are returned by the function.
+/* BMK_benchFunctionTimed() :
+ * Same as BMK_benchFunction(), but runs for nbSeconds seconds rather than a fixed number of loops.
+ * Arguments are mostly the same other as BMK_benchFunction()
+ * Usage - benchFunctionTimed will return in approximately one second.
+ * Keep calling BMK_benchFunctionTimed() until @return.completed == 1,
+ * to continue updating intermediate result.
+ * Intermediate return values are returned by the function.
  */
 BMK_customTimedReturn_t BMK_benchFunctionTimed(BMK_timedFnState_t* cont,
     BMK_benchFn_t benchFn, void* benchPayload,
     BMK_initFn_t initFn, void* initPayload,
     size_t blockCount,
-    const void* const * const srcBlockBuffers, const size_t* srcBlockSizes,
-    void* const * const dstBlockBuffers, const size_t* dstBlockCapacities, size_t* blockResults);
+    const void *const * srcBlockBuffers, const size_t* srcBlockSizes,
+    void *const * dstBlockBuffers, const size_t* dstBlockCapacities, size_t* blockResults);
 
 #endif   /* BENCH_H_121279284357 */
 
diff --git a/tests/paramgrill.c b/tests/paramgrill.c
index f42afe403..1781bf7ff 100644
--- a/tests/paramgrill.c
+++ b/tests/paramgrill.c
@@ -68,7 +68,7 @@ static const int g_maxNbVariations = 64;
 #define FADT_MIN 0
 #define FADT_MAX ((U32)-1)
 
-#define ZSTD_TARGETLENGTH_MIN 0 
+#define ZSTD_TARGETLENGTH_MIN 0
 #define ZSTD_TARGETLENGTH_MAX 999
 
 #define WLOG_RANGE (ZSTD_WINDOWLOG_MAX - ZSTD_WINDOWLOG_MIN + 1)
@@ -115,27 +115,27 @@ typedef struct {
 } paramValues_t;
 
 /* maximum value of parameters */
-static const U32 mintable[NUM_PARAMS] = 
+static const U32 mintable[NUM_PARAMS] =
         { ZSTD_WINDOWLOG_MIN, ZSTD_CHAINLOG_MIN, ZSTD_HASHLOG_MIN, ZSTD_SEARCHLOG_MIN, ZSTD_SEARCHLENGTH_MIN, ZSTD_TARGETLENGTH_MIN, ZSTD_fast, FADT_MIN };
 
 /* minimum value of parameters */
-static const U32 maxtable[NUM_PARAMS] = 
+static const U32 maxtable[NUM_PARAMS] =
         { ZSTD_WINDOWLOG_MAX, ZSTD_CHAINLOG_MAX, ZSTD_HASHLOG_MAX, ZSTD_SEARCHLOG_MAX, ZSTD_SEARCHLENGTH_MAX, ZSTD_TARGETLENGTH_MAX, ZSTD_btultra, FADT_MAX };
 
 /* # of values parameters can take on */
-static const U32 rangetable[NUM_PARAMS] = 
+static const U32 rangetable[NUM_PARAMS] =
         { WLOG_RANGE, CLOG_RANGE, HLOG_RANGE, SLOG_RANGE, SLEN_RANGE, TLEN_RANGE, STRT_RANGE, FADT_RANGE };
 
 /* ZSTD_cctxSetParameter() index to set */
-static const ZSTD_cParameter cctxSetParamTable[NUM_PARAMS] = 
+static const ZSTD_cParameter cctxSetParamTable[NUM_PARAMS] =
         { ZSTD_p_windowLog, ZSTD_p_chainLog, ZSTD_p_hashLog, ZSTD_p_searchLog, ZSTD_p_minMatch, ZSTD_p_targetLength, ZSTD_p_compressionStrategy, ZSTD_p_forceAttachDict };
 
 /* names of parameters */
-static const char* g_paramNames[NUM_PARAMS] = 
+static const char* g_paramNames[NUM_PARAMS] =
         { "windowLog", "chainLog", "hashLog","searchLog", "searchLength", "targetLength", "strategy", "forceAttachDict" };
 
 /* shortened names of parameters */
-static const char* g_shortParamNames[NUM_PARAMS] = 
+static const char* g_shortParamNames[NUM_PARAMS] =
         { "wlog", "clog", "hlog","slog", "slen", "tlen", "strt", "fadt" };
 
 /* maps value from { 0 to rangetable[param] - 1 } to valid paramvalues */
@@ -178,7 +178,7 @@ static int invRangeMap(varInds_t param, U32 value) {
                     hi = mid;
                 }
             }
-            return lo;    
+            return lo;
         }
         case fadt_ind:
             return (int)value + 1;
@@ -201,11 +201,11 @@ static void displayParamVal(FILE* f, varInds_t param, U32 value, int width) {
     switch(param) {
         case fadt_ind: if(width) { fprintf(f, "%*d", width, (int)value); } else { fprintf(f, "%d", (int)value); } break;
         case strt_ind: if(width) { fprintf(f, "%*s", width, g_stratName[value]); } else { fprintf(f, "%s", g_stratName[value]); } break;
-        case wlog_ind: 
-        case clog_ind: 
-        case hlog_ind: 
-        case slog_ind: 
-        case slen_ind: 
+        case wlog_ind:
+        case clog_ind:
+        case hlog_ind:
+        case slog_ind:
+        case slen_ind:
         case tlen_ind: if(width) { fprintf(f, "%*u", width, value); } else { fprintf(f, "%u", value); } break;
         case NUM_PARAMS:
             DISPLAY("Error, not a valid param\n "); break;
@@ -265,7 +265,7 @@ typedef struct {
 typedef struct {
     U32 cSpeed;  /* bytes / sec */
     U32 dSpeed;
-    U32 cMem;    /* bytes */    
+    U32 cMem;    /* bytes */
 } constraint_t;
 
 typedef struct winner_ll_node winner_ll_node;
@@ -285,7 +285,7 @@ static winner_ll_node* g_winners; /* linked list sorted ascending by cSize & cSp
  */
 
 /*-*******************************************************
-*  General Util Functions 
+*  General Util Functions
 *********************************************************/
 
 /* nullified useless params, to ensure count stats */
@@ -502,10 +502,10 @@ static int feasible(const BMK_result_t results, const constraint_t target) {
 }
 
 /* hill climbing value for part 1 */
-/* Scoring here is a linear reward for all set constraints normalized between 0 to 1 
- * (with 0 at 0 and 1 being fully fulfilling the constraint), summed with a logarithmic 
+/* Scoring here is a linear reward for all set constraints normalized between 0 to 1
+ * (with 0 at 0 and 1 being fully fulfilling the constraint), summed with a logarithmic
  * bonus to exceeding the constraint value. We also give linear ratio for compression ratio.
- * The constant factors are experimental. 
+ * The constant factors are experimental.
  */
 static double resultScore(const BMK_result_t res, const size_t srcSize, const constraint_t target) {
     double cs = 0., ds = 0., rt, cm = 0.;
@@ -516,7 +516,7 @@ static double resultScore(const BMK_result_t res, const size_t srcSize, const co
     if(target.cMem != (U32)-1) { cm = (double)target.cMem / res.cMem; }
     rt = ((double)srcSize / res.cSize);
 
-    ret = (MIN(1, cs) + MIN(1, ds)  + MIN(1, cm))*r1 + rt * rtr + 
+    ret = (MIN(1, cs) + MIN(1, ds)  + MIN(1, cm))*r1 + rt * rtr +
          (MAX(0, log(cs))+ MAX(0, log(ds))+ MAX(0, log(cm))) * r2;
 
     return ret;
@@ -532,7 +532,7 @@ static double resultDistLvl(const BMK_result_t result1, const BMK_result_t lvlRe
     return normalizedRatioGain1 * g_ratioMultiplier + normalizedCSpeedGain1;
 }
 
-/* return true if r2 strictly better than r1 */ 
+/* return true if r2 strictly better than r1 */
 static int compareResultLT(const BMK_result_t result1, const BMK_result_t result2, const constraint_t target, size_t srcSize) {
     if(feasible(result1, target) && feasible(result2, target)) {
         if(g_optmode) {
@@ -547,7 +547,7 @@ static int compareResultLT(const BMK_result_t result1, const BMK_result_t result
 
 static constraint_t relaxTarget(constraint_t target) {
     target.cMem = (U32)-1;
-    target.cSpeed *= ((double)g_strictness) / 100; 
+    target.cSpeed *= ((double)g_strictness) / 100;
     target.dSpeed *= ((double)g_strictness) / 100;
     return target;
 }
@@ -598,7 +598,7 @@ static void optimizerAdjustInput(paramValues_t* pc, const size_t maxBlockSize) {
             DISPLAY("Warning: hashlog too much larger than windowLog size, adjusted to %u\n", pc->vals[hlog_ind]);
         }
     }
-    
+
     if(pc->vals[slog_ind] != PARAM_UNSET && pc->vals[clog_ind] != PARAM_UNSET) {
         if(pc->vals[slog_ind] > pc->vals[clog_ind]) {
             pc->vals[clog_ind] = pc->vals[slog_ind];
@@ -608,17 +608,17 @@ static void optimizerAdjustInput(paramValues_t* pc, const size_t maxBlockSize) {
 }
 
 static int redundantParams(const paramValues_t paramValues, const constraint_t target, const size_t maxBlockSize) {
-    return 
+    return
        (ZSTD_estimateCStreamSize_usingCParams(pvalsToCParams(paramValues)) > (size_t)target.cMem) /* Uses too much memory */
     || ((1ULL << (paramValues.vals[wlog_ind] - 1)) >= maxBlockSize && paramValues.vals[wlog_ind] != mintable[wlog_ind]) /* wlog too much bigger than src size */
     || (paramValues.vals[clog_ind] > (paramValues.vals[wlog_ind] + (paramValues.vals[strt_ind] > ZSTD_btlazy2))) /* chainLog larger than windowLog*/
     || (paramValues.vals[slog_ind] > paramValues.vals[clog_ind]) /* searchLog larger than chainLog */
     || (paramValues.vals[hlog_ind] > paramValues.vals[wlog_ind] + 1); /* hashLog larger than windowLog + 1 */
-    
+
 }
 
 /*-************************************
-*  Display Functions 
+*  Display Functions
 **************************************/
 
 static void BMK_translateAdvancedParams(FILE* f, const paramValues_t params) {
@@ -629,7 +629,7 @@ static void BMK_translateAdvancedParams(FILE* f, const paramValues_t params) {
         if(g_silenceParams[v]) { continue; }
         if(!first) { fprintf(f, ","); }
         fprintf(f,"%s=", g_paramNames[v]);
-        
+
         if(v == strt_ind) { fprintf(f,"%u", params.vals[v]); }
         else { displayParamVal(f, v, params.vals[v], 0); }
         first = 0;
@@ -668,10 +668,10 @@ static void BMK_printWinner(FILE* f, const int cLevel, const BMK_result_t result
         snprintf(lvlstr, 15, "  Level %2d  ", cLevel);
     }
 
-    if(TIMED) { 
+    if(TIMED) {
         const U64 time = UTIL_clockSpanNano(g_time);
         const U64 minutes = time / (60ULL * TIMELOOP_NANOSEC);
-        fprintf(f, "%1lu:%2lu:%05.2f - ", (unsigned long) minutes / 60,(unsigned long) minutes % 60, (double)(time - minutes * TIMELOOP_NANOSEC * 60ULL)/TIMELOOP_NANOSEC); 
+        fprintf(f, "%1lu:%2lu:%05.2f - ", (unsigned long) minutes / 60,(unsigned long) minutes % 60, (double)(time - minutes * TIMELOOP_NANOSEC * 60ULL)/TIMELOOP_NANOSEC);
     }
 
     fprintf(f, "/* %s */   ", lvlstr);
@@ -735,7 +735,7 @@ static int insertWinner(const winnerInfo_t w, const constraint_t targetConstrain
                 tmp = cur_node->next;
                 cur_node->next = cur_node->next->next;
                 free(tmp);
-                break; 
+                break;
             }
             case SIZE_RESULT:
             {
@@ -754,7 +754,7 @@ static int insertWinner(const winnerInfo_t w, const constraint_t targetConstrain
                 cur_node->next = newnode;
                 return 0;
             }
-        } 
+        }
 
     }
 
@@ -792,9 +792,9 @@ static int insertWinner(const winnerInfo_t w, const constraint_t targetConstrain
             cur_node->next = newnode;
             return 0;
         }
-        default: 
+        default:
             return 1;
-    } 
+    }
 }
 
 static void BMK_printWinnerOpt(FILE* f, const U32 cLevel, const BMK_result_t result, const paramValues_t params, const constraint_t targetConstraints, const size_t srcSize)
@@ -814,7 +814,7 @@ static void BMK_printWinnerOpt(FILE* f, const U32 cLevel, const BMK_result_t res
             g_winner.result = result;
             g_winner.params = params;
         }
-    }  
+    }
 
     if(g_optmode && g_optimizer && (DEBUG || g_displayLevel == 3)) {
         winnerInfo_t w;
@@ -824,8 +824,8 @@ static void BMK_printWinnerOpt(FILE* f, const U32 cLevel, const BMK_result_t res
         insertWinner(w, targetConstraints);
 
         if(!DEBUG) { fprintf(f, "\033c"); }
-        fprintf(f, "\n"); 
-        
+        fprintf(f, "\n");
+
         /* the table */
         fprintf(f, "================================\n");
         for(n = g_winners; n != NULL; n = n->next) {
@@ -909,8 +909,8 @@ static size_t local_initDCtx(void* payload) {
 
 /* additional argument is just the context */
 static size_t local_defaultCompress(
-    const void* srcBuffer, size_t srcSize, 
-    void* dstBuffer, size_t dstSize, 
+    const void* srcBuffer, size_t srcSize,
+    void* dstBuffer, size_t dstSize,
     void* addArgs) {
     size_t moreToFlush = 1;
     ZSTD_CCtx* ctx = (ZSTD_CCtx*)addArgs;
@@ -937,8 +937,8 @@ static size_t local_defaultCompress(
 
 /* additional argument is just the context */
 static size_t local_defaultDecompress(
-    const void* srcBuffer, size_t srcSize, 
-    void* dstBuffer, size_t dstSize, 
+    const void* srcBuffer, size_t srcSize,
+    void* dstBuffer, size_t dstSize,
     void* addArgs) {
     size_t moreToFlush = 1;
     ZSTD_DCtx* dctx = (ZSTD_DCtx*)addArgs;
@@ -965,7 +965,7 @@ static size_t local_defaultDecompress(
 }
 
 /*-************************************
-*  Data Initialization Functions 
+*  Data Initialization Functions
 **************************************/
 
 typedef struct {
@@ -1041,7 +1041,7 @@ static int createBuffersFromMemory(buffers_t* buff, void * srcBuffer, const size
     buff->dstSizes = (size_t*)malloc(maxNbBlocks * sizeof(size_t));
 
     buff->resPtrs = (void**)calloc(maxNbBlocks, sizeof(void*));
-    buff->resSizes = (size_t*)malloc(maxNbBlocks * sizeof(size_t)); 
+    buff->resSizes = (size_t*)malloc(maxNbBlocks * sizeof(size_t));
 
     if(!buff->srcPtrs || !buff->srcSizes || !buff->dstPtrs || !buff->dstCapacities || !buff->dstSizes || !buff->resPtrs || !buff->resSizes) {
         DISPLAY("alloc error\n");
@@ -1090,18 +1090,18 @@ static int createBuffersFromMemory(buffers_t* buff, void * srcBuffer, const size
     buff->nbBlocks = blockNb;
 
     return 0;
-} 
+}
 
 /* allocates buffer's arguments. returns success / failuere */
-static int createBuffers(buffers_t* buff, const char* const * const fileNamesTable, 
+static int createBuffers(buffers_t* buff, const char* const * const fileNamesTable,
                           size_t nbFiles) {
     size_t pos = 0;
     size_t n;
     size_t totalSizeToLoad = UTIL_getTotalFileSize(fileNamesTable, (U32)nbFiles);
     size_t benchedSize = MIN(BMK_findMaxMem(totalSizeToLoad * 3) / 3, totalSizeToLoad);
     size_t* fileSizes = calloc(sizeof(size_t), nbFiles);
-    void* srcBuffer = NULL; 
-    int ret = 0;  
+    void* srcBuffer = NULL;
+    int ret = 0;
 
     if(!totalSizeToLoad || !benchedSize) {
         ret = 1;
@@ -1139,7 +1139,7 @@ static int createBuffers(buffers_t* buff, const char* const * const fileNamesTab
 
         if (fileSize + pos > benchedSize) fileSize = benchedSize - pos, nbFiles=n;   /* buffer too small - stop after this file */
         {
-            char* buffer = (char*)(srcBuffer); 
+            char* buffer = (char*)(srcBuffer);
             size_t const readSize = fread((buffer)+pos, 1, (size_t)fileSize, f);
             fclose(f);
             if (readSize != (size_t)fileSize) {
@@ -1181,14 +1181,14 @@ static int createContexts(contexts_t* ctx, const char* dictFileName) {
     ctx->dictBuffer = malloc(ctx->dictSize);
 
     f = fopen(dictFileName, "rb");
-    
+
     if(!f) {
         DISPLAY("unable to open file\n");
         fclose(f);
         freeContexts(*ctx);
         return 1;
     }
-    
+
     if(ctx->dictSize > 64 MB || !(ctx->dictBuffer)) {
         DISPLAY("dictionary too large\n");
         fclose(f);
@@ -1207,7 +1207,7 @@ static int createContexts(contexts_t* ctx, const char* dictFileName) {
 }
 
 /*-************************************
-*  Optimizer Memoization Functions 
+*  Optimizer Memoization Functions
 **************************************/
 
 /* return: new length */
@@ -1218,7 +1218,7 @@ static size_t sanitizeVarArray(varInds_t* varNew, const size_t varLength, const
     for(i = 0; i < varLength; i++) {
         if( !((varArray[i] == clog_ind && strat == ZSTD_fast)
             || (varArray[i] == slog_ind && strat == ZSTD_fast)
-            || (varArray[i] == slog_ind && strat == ZSTD_dfast) 
+            || (varArray[i] == slog_ind && strat == ZSTD_dfast)
             || (varArray[i] == tlen_ind && strat != ZSTD_btopt && strat != ZSTD_btultra && strat != ZSTD_fast))) {
             varNew[j] = varArray[i];
             j++;
@@ -1305,7 +1305,7 @@ static void freeMemoTableArray(memoTable_t* const mtAll) {
 static memoTable_t* createMemoTableArray(const paramValues_t p, const varInds_t* const varyParams, const size_t varyLen, const U32 memoTableLog) {
     memoTable_t* mtAll = (memoTable_t*)calloc(sizeof(memoTable_t),(ZSTD_btultra + 1));
     ZSTD_strategy i, stratMin = ZSTD_fast, stratMax = ZSTD_btultra;
-    
+
     if(mtAll == NULL) {
         return NULL;
     }
@@ -1324,7 +1324,7 @@ static memoTable_t* createMemoTableArray(const paramValues_t p, const varInds_t*
         return mtAll;
     }
 
-    
+
     if(p.vals[strt_ind] != PARAM_UNSET) {
         stratMin = p.vals[strt_ind];
         stratMax = p.vals[strt_ind];
@@ -1348,7 +1348,7 @@ static memoTable_t* createMemoTableArray(const paramValues_t p, const varInds_t*
             return NULL;
         }
     }
-    
+
     return mtAll;
 }
 
@@ -1363,7 +1363,7 @@ static void randomConstrainedParams(paramValues_t* pc, const memoTable_t* memoTa
         int i;
         for(i = 0; i < NUM_PARAMS; i++) {
             varInds_t v = mt.varArray[i];
-            if(v == strt_ind) continue; 
+            if(v == strt_ind) continue;
             pc->vals[v] = rangeMap(v, FUZ_rand(&g_rand) % rangetable[v]);
         }
 
@@ -1382,7 +1382,7 @@ static void randomConstrainedParams(paramValues_t* pc, const memoTable_t* memoTa
 
 /* if in decodeOnly, then srcPtr's will be compressed blocks, and uncompressedBlocks will be written to dstPtrs */
 /* dictionary nullable, nothing else though. */
-static BMK_return_t BMK_benchMemInvertible(const buffers_t buf, const contexts_t ctx, 
+static BMK_return_t BMK_benchMemInvertible(const buffers_t buf, const contexts_t ctx,
                         const int cLevel, const paramValues_t* comprParams,
                         const BMK_mode_t mode, const BMK_loopMode_t loopMode, const unsigned nbSeconds) {
 
@@ -1427,9 +1427,9 @@ static BMK_return_t BMK_benchMemInvertible(const buffers_t buf, const contexts_t
 
         if(loopMode == BMK_timeMode) {
             BMK_customTimedReturn_t intermediateResultCompress;
-            BMK_customTimedReturn_t intermediateResultDecompress;  
-            BMK_timedFnState_t* timeStateCompress = BMK_createTimeState(nbSeconds);
-            BMK_timedFnState_t* timeStateDecompress = BMK_createTimeState(nbSeconds);
+            BMK_customTimedReturn_t intermediateResultDecompress;
+            BMK_timedFnState_t* timeStateCompress = BMK_createTimedFnState(nbSeconds);
+            BMK_timedFnState_t* timeStateDecompress = BMK_createTimedFnState(nbSeconds);
             if(mode == BMK_compressOnly) {
                 intermediateResultCompress.completed = 0;
                 intermediateResultDecompress.completed = 1;
@@ -1447,8 +1447,8 @@ static BMK_return_t BMK_benchMemInvertible(const buffers_t buf, const contexts_t
 
                 if(intermediateResultCompress.result.error) {
                     results.error = intermediateResultCompress.result.error;
-                    BMK_freeTimeState(timeStateCompress);
-                    BMK_freeTimeState(timeStateDecompress);
+                    BMK_freeTimedFnState(timeStateCompress);
+                    BMK_freeTimedFnState(timeStateDecompress);
                     return results;
                 }
                 results.result.cSpeed = (srcSize * TIMELOOP_NANOSEC) / intermediateResultCompress.result.result.nanoSecPerRun;
@@ -1461,21 +1461,21 @@ static BMK_return_t BMK_benchMemInvertible(const buffers_t buf, const contexts_t
 
                 if(intermediateResultDecompress.result.error) {
                     results.error = intermediateResultDecompress.result.error;
-                    BMK_freeTimeState(timeStateCompress);
-                    BMK_freeTimeState(timeStateDecompress);
+                    BMK_freeTimedFnState(timeStateCompress);
+                    BMK_freeTimedFnState(timeStateDecompress);
                     return results;
                 }
                 results.result.dSpeed = (srcSize * TIMELOOP_NANOSEC) / intermediateResultDecompress.result.result.nanoSecPerRun;
             }
 
-            BMK_freeTimeState(timeStateCompress);
-            BMK_freeTimeState(timeStateDecompress);
+            BMK_freeTimedFnState(timeStateCompress);
+            BMK_freeTimedFnState(timeStateDecompress);
 
         } else { /* iterMode; */
             if(mode != BMK_decodeOnly) {
 
                 BMK_customReturn_t compressionResults = BMK_benchFunction(&local_defaultCompress, (void*)cctx, &local_initCCtx, (void*)&cctxprep,
-                    nbBlocks, srcPtrs, srcSizes, dstPtrs, dstCapacities, dstSizes, nbSeconds); 
+                    nbBlocks, srcPtrs, srcSizes, dstPtrs, dstCapacities, dstSizes, nbSeconds);
                 if(compressionResults.error) {
                     results.error = compressionResults.error;
                     return results;
@@ -1537,7 +1537,7 @@ static int BMK_benchParam(BMK_result_t* resultPtr,
 }
 
 /* Benchmarking which stops when we are sufficiently sure the solution is infeasible / worse than the winner */
-#define VARIANCE 1.2 
+#define VARIANCE 1.2
 static int allBench(BMK_result_t* resultPtr,
                 const buffers_t buf, const contexts_t ctx,
                 const paramValues_t cParams,
@@ -1557,13 +1557,13 @@ static int allBench(BMK_result_t* resultPtr,
 
     /* calculate uncertainty in compression / decompression runs */
     if(benchres.cSpeed) {
-        loopDurationC = ((buf.srcSize * TIMELOOP_NANOSEC) / benchres.cSpeed); 
-        uncertaintyConstantC = ((loopDurationC + (double)(2 * g_clockGranularity))/loopDurationC); 
+        loopDurationC = ((buf.srcSize * TIMELOOP_NANOSEC) / benchres.cSpeed);
+        uncertaintyConstantC = ((loopDurationC + (double)(2 * g_clockGranularity))/loopDurationC);
     }
 
     if(benchres.dSpeed) {
-        loopDurationD = ((buf.srcSize * TIMELOOP_NANOSEC) / benchres.dSpeed); 
-        uncertaintyConstantD = ((loopDurationD + (double)(2 * g_clockGranularity))/loopDurationD);  
+        loopDurationD = ((buf.srcSize * TIMELOOP_NANOSEC) / benchres.dSpeed);
+        uncertaintyConstantD = ((loopDurationD + (double)(2 * g_clockGranularity))/loopDurationD);
     }
 
     /* anything with worse ratio in feas is definitely worse, discard */
@@ -1596,18 +1596,18 @@ static int allBench(BMK_result_t* resultPtr,
 
     /* compare by resultScore when in infeas */
     /* compare by compareResultLT when in feas */
-    if((!feas && (resultScore(benchres, buf.srcSize, target) > resultScore(*winnerResult, buf.srcSize, target))) || 
-       (feas && (compareResultLT(*winnerResult, benchres, target, buf.srcSize))) )  { 
-        return BETTER_RESULT; 
-    } else { 
-        return WORSE_RESULT; 
+    if((!feas && (resultScore(benchres, buf.srcSize, target) > resultScore(*winnerResult, buf.srcSize, target))) ||
+       (feas && (compareResultLT(*winnerResult, benchres, target, buf.srcSize))) )  {
+        return BETTER_RESULT;
+    } else {
+        return WORSE_RESULT;
     }
 }
 
 #define INFEASIBLE_THRESHOLD 200
 /* Memoized benchmarking, won't benchmark anything which has already been benchmarked before. */
 static int benchMemo(BMK_result_t* resultPtr,
-                const buffers_t buf, const contexts_t ctx, 
+                const buffers_t buf, const contexts_t ctx,
                 const paramValues_t cParams,
                 const constraint_t target,
                 BMK_result_t* winnerResult, memoTable_t* const memoTableArray,
@@ -1615,7 +1615,7 @@ static int benchMemo(BMK_result_t* resultPtr,
     static int bmcount = 0;
     int res;
 
-    if(memoTableGet(memoTableArray, cParams) >= INFEASIBLE_THRESHOLD || redundantParams(cParams, target, buf.maxBlockSize)) { return WORSE_RESULT; } 
+    if(memoTableGet(memoTableArray, cParams) >= INFEASIBLE_THRESHOLD || redundantParams(cParams, target, buf.maxBlockSize)) { return WORSE_RESULT; }
 
     res = allBench(resultPtr, buf, ctx, cParams, target, winnerResult, feas);
 
@@ -1659,7 +1659,7 @@ static void BMK_init_level_constraints(int bytePerSec_level1)
     }   }
 }
 
-static int BMK_seed(winnerInfo_t* winners, const paramValues_t params, 
+static int BMK_seed(winnerInfo_t* winners, const paramValues_t params,
                     const buffers_t buf, const contexts_t ctx)
 {
     BMK_result_t testResult;
@@ -1781,7 +1781,7 @@ static void playAround(FILE* f, winnerInfo_t* winners,
 
         if (nbVariations++ > g_maxNbVariations) break;
 
-        do { for(i = 0; i < 4; i++) { paramVaryOnce(FUZ_rand(&g_rand) % (strt_ind + 1), ((FUZ_rand(&g_rand) & 1) << 1) - 1, &p); } } 
+        do { for(i = 0; i < 4; i++) { paramVaryOnce(FUZ_rand(&g_rand) % (strt_ind + 1), ((FUZ_rand(&g_rand) & 1) << 1) - 1, &p); } }
         while(!paramValid(p));
 
         /* exclude faster if already played params */
@@ -1815,7 +1815,7 @@ static void BMK_selectRandomStart(
     }
 }
 
-static void BMK_benchFullTable(const buffers_t buf, const contexts_t ctx) 
+static void BMK_benchFullTable(const buffers_t buf, const contexts_t ctx)
 {
     paramValues_t params;
     winnerInfo_t winners[NB_LEVELS_TRACKED+1];
@@ -1888,7 +1888,7 @@ static int benchSample(double compressibility, int cLevel)
 
     buffers_t buf;
     contexts_t ctx;
-    
+
     if(srcBuffer == NULL) {
         DISPLAY("Out of Memory\n");
         return 2;
@@ -1967,12 +1967,12 @@ int benchFiles(const char** fileNamesTable, int nbFiles, const char* dictFileNam
 *  Local Optimization Functions
 **************************************/
 
-/* One iteration of hill climbing. Specifically, it first tries all 
+/* One iteration of hill climbing. Specifically, it first tries all
  * valid parameter configurations w/ manhattan distance 1 and picks the best one
  * failing that, it progressively tries candidates further and further away (up to #dim + 2)
- * if it finds a candidate exceeding winnerInfo, it will repeat. Otherwise, it will stop the 
- * current stage of hill climbing. 
- * Each iteration of hill climbing proceeds in 2 'phases'. Phase 1 climbs according to 
+ * if it finds a candidate exceeding winnerInfo, it will repeat. Otherwise, it will stop the
+ * current stage of hill climbing.
+ * Each iteration of hill climbing proceeds in 2 'phases'. Phase 1 climbs according to
  * the resultScore function, which is effectively a linear increase in reward until it reaches
  * the constraint-satisfying value, it which point any excess results in only logarithmic reward.
  * This aims to find some constraint-satisfying point.
@@ -1980,14 +1980,14 @@ int benchFiles(const char** fileNamesTable, int nbFiles, const char* dictFileNam
  * all feasible solutions valued over all infeasible solutions.
  */
 
-/* sanitize all params here. 
+/* sanitize all params here.
  * all generation after random should be sanitized. (maybe sanitize random)
  */
-static winnerInfo_t climbOnce(const constraint_t target, 
-                memoTable_t* mtAll, 
+static winnerInfo_t climbOnce(const constraint_t target,
+                memoTable_t* mtAll,
                 const buffers_t buf, const contexts_t ctx,
                 const paramValues_t init) {
-    /* 
+    /*
      * cparam - currently considered 'center'
      * candidate - params to benchmark/results
      * winner - best option found so far.
@@ -2017,8 +2017,8 @@ static winnerInfo_t climbOnce(const constraint_t target,
                 for(offset = -1; offset <= 1; offset += 2) {
                     CHECKTIME(winnerInfo);
                     candidateInfo.params = cparam;
-                    paramVaryOnce(mtAll[cparam.vals[strt_ind]].varArray[i], offset, &candidateInfo.params); 
-                    
+                    paramVaryOnce(mtAll[cparam.vals[strt_ind]].varArray[i], offset, &candidateInfo.params);
+
                     if(paramValid(candidateInfo.params)) {
                         int res;
                         res = benchMemo(&candidateInfo.result, buf, ctx,
@@ -2047,7 +2047,7 @@ static winnerInfo_t climbOnce(const constraint_t target,
                     /* param error checking already done here */
                     paramVariation(&candidateInfo.params, mtAll, (U32)dist);
 
-                    res = benchMemo(&candidateInfo.result, buf, ctx, 
+                    res = benchMemo(&candidateInfo.result, buf, ctx,
                         sanitizeParams(candidateInfo.params), target, &winnerInfo.result, mtAll, feas);
                     DEBUGOUTPUT("Res: %d\n", res);
                     if(res == BETTER_RESULT) { /* synonymous with better in this case*/
@@ -2065,8 +2065,8 @@ static winnerInfo_t climbOnce(const constraint_t target,
                 }
             }
 
-            if(!better) { /* infeas -> feas -> stop */ 
-                if(feas) { return winnerInfo; } 
+            if(!better) { /* infeas -> feas -> stop */
+                if(feas) { return winnerInfo; }
 
                 feas = 1;
                 better = 1;
@@ -2084,17 +2084,17 @@ static winnerInfo_t climbOnce(const constraint_t target,
 
 /* flexible parameters: iterations of failed climbing (or if we do non-random, maybe this is when everything is close to visitied)
    weight more on visit for bad results, less on good results/more on later results / ones with more failures.
-   allocate memoTable here. 
+   allocate memoTable here.
  */
 static winnerInfo_t optimizeFixedStrategy(
-    const buffers_t buf, const contexts_t ctx, 
+    const buffers_t buf, const contexts_t ctx,
     const constraint_t target, paramValues_t paramTarget,
-    const ZSTD_strategy strat, 
+    const ZSTD_strategy strat,
     memoTable_t* memoTableArray, const int tries) {
     int i = 0;
 
     paramValues_t init;
-    winnerInfo_t winnerInfo, candidateInfo; 
+    winnerInfo_t winnerInfo, candidateInfo;
     winnerInfo = initWinnerInfo(emptyParams());
     /* so climb is given the right fixed strategy */
     paramTarget.vals[strt_ind] = strat;
@@ -2104,7 +2104,7 @@ static winnerInfo_t optimizeFixedStrategy(
     init = paramTarget;
 
     for(i = 0; i < tries; i++) {
-        DEBUGOUTPUT("Restart\n"); 
+        DEBUGOUTPUT("Restart\n");
         do { randomConstrainedParams(&init, memoTableArray, strat); } while(redundantParams(init, target, buf.maxBlockSize));
         candidateInfo = climbOnce(target, memoTableArray, buf, ctx, init);
         if(compareResultLT(winnerInfo.result, candidateInfo.result, target, buf.srcSize)) {
@@ -2153,9 +2153,9 @@ static int nextStrategy(const int currentStrategy, const int bestStrategy) {
 
 /* main fn called when using --optimize */
 /* Does strategy selection by benchmarking default compression levels
- * then optimizes by strategy, starting with the best one and moving 
+ * then optimizes by strategy, starting with the best one and moving
  * progressively moving further away by number
- * args: 
+ * args:
  * fileNamesTable - list of files to benchmark
  * nbFiles - length of fileNamesTable
  * dictFileName - name of dictionary file if one, else NULL
@@ -2167,7 +2167,7 @@ static int nextStrategy(const int currentStrategy, const int bestStrategy) {
 static int g_maxTries = 5;
 #define TRY_DECAY 1
 
-static int optimizeForSize(const char* const * const fileNamesTable, const size_t nbFiles, const char* dictFileName, constraint_t target, paramValues_t paramTarget, 
+static int optimizeForSize(const char* const * const fileNamesTable, const size_t nbFiles, const char* dictFileName, constraint_t target, paramValues_t paramTarget,
     const int cLevelOpt, const int cLevelRun, const U32 memoTableLog)
 {
     varInds_t varArray [NUM_PARAMS];
@@ -2194,7 +2194,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_
     if(nbFiles == 1) {
         DISPLAYLEVEL(2, "Loading %s...       \r", fileNamesTable[0]);
     } else {
-        DISPLAYLEVEL(2, "Loading %lu Files...       \r", (unsigned long)nbFiles); 
+        DISPLAYLEVEL(2, "Loading %lu Files...       \r", (unsigned long)nbFiles);
     }
 
     /* sanitize paramTarget */
@@ -2231,14 +2231,14 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_
             ret = 3;
             goto _cleanUp;
         }
- 
-        g_lvltarget = winner.result; 
+
+        g_lvltarget = winner.result;
         g_lvltarget.cSpeed *= ((double)g_strictness) / 100;
         g_lvltarget.dSpeed *= ((double)g_strictness) / 100;
         g_lvltarget.cSize /= ((double)g_strictness) / 100;
 
-        target.cSpeed = (U32)g_lvltarget.cSpeed;  
-        target.dSpeed = (U32)g_lvltarget.dSpeed; 
+        target.cSpeed = (U32)g_lvltarget.cSpeed;
+        target.dSpeed = (U32)g_lvltarget.dSpeed;
 
         BMK_printWinnerOpt(stdout, cLevelOpt, winner.result, winner.params, target, buf.srcSize);
     }
@@ -2272,7 +2272,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_
     DISPLAYLEVEL(2, "\n");
     findClockGranularity();
 
-    {   
+    {
         paramValues_t CParams;
 
         /* find best solution from default params */
@@ -2305,13 +2305,13 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_
 
         DEBUGOUTPUT("Real Opt\n");
         /* start 'real' optimization */
-        {   
+        {
             int bestStrategy = (int)winner.params.vals[strt_ind];
             if(paramTarget.vals[strt_ind] == PARAM_UNSET) {
                 int st = bestStrategy;
                 int tries = g_maxTries;
 
-                { 
+                {
                     /* one iterations of hill climbing with the level-defined parameters. */
                     winnerInfo_t w1 = climbOnce(target, allMT, buf, ctx, winner.params);
                     if(compareResultLT(winner.result, w1.result, target, buf.srcSize)) {
@@ -2323,7 +2323,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_
                 while(st && tries > 0) {
                     winnerInfo_t wc;
                     DEBUGOUTPUT("StrategySwitch: %s\n", g_stratName[st]);
-                    
+
                     wc = optimizeFixedStrategy(buf, ctx, target, paramBase, st, allMT, tries);
 
                     if(compareResultLT(winner.result, wc.result, target, buf.srcSize)) {
@@ -2349,7 +2349,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_
             goto _cleanUp;
         }
         /* end summary */
-_displayCleanUp: 
+_displayCleanUp:
         if(g_displayLevel >= 0) { BMK_displayOneResult(stdout, winner, buf.srcSize); }
         BMK_translateAdvancedParams(stdout, winner.params);
         DISPLAYLEVEL(1, "grillParams size - optimizer completed \n");
@@ -2427,7 +2427,7 @@ static double readDoubleFromChar(const char** stringPtr)
     }
     (*stringPtr)++;
     while ((**stringPtr >='0') && (**stringPtr <='9')) {
-        result += (double)(**stringPtr - '0') / divide, divide *= 10, (*stringPtr)++ ; 
+        result += (double)(**stringPtr - '0') / divide, divide *= 10, (*stringPtr)++ ;
     }
     return result;
 }
@@ -2503,7 +2503,7 @@ int main(int argc, const char** argv)
     int seperateFiles = 0;
     double compressibility = COMPRESSIBILITY_DEFAULT;
     U32 memoTableLog = PARAM_UNSET;
-    constraint_t target = { 0, 0, (U32)-1 }; 
+    constraint_t target = { 0, 0, (U32)-1 };
 
     paramValues_t paramTarget = emptyParams();
     g_params = emptyParams();
@@ -2522,7 +2522,7 @@ int main(int argc, const char** argv)
             for ( ; ;) {
                 if(parse_params(&argument, ¶mTarget)) { if(argument[0] == ',') { argument++; continue; } else break; }
                 PARSE_SUB_ARGS("compressionSpeed=" ,  "cSpeed=", target.cSpeed);
-                PARSE_SUB_ARGS("decompressionSpeed=", "dSpeed=", target.dSpeed); 
+                PARSE_SUB_ARGS("decompressionSpeed=", "dSpeed=", target.dSpeed);
                 PARSE_SUB_ARGS("compressionMemory=" , "cMem=", target.cMem);
                 PARSE_SUB_ARGS("strict=", "stc=", g_strictness);
                 PARSE_SUB_ARGS("maxTries=", "tries=", g_maxTries);
@@ -2701,7 +2701,7 @@ int main(int argc, const char** argv)
 
                 /* load dictionary file (only applicable for optimizer rn) */
                 case 'D':
-                    if(i == argc - 1) { /* last argument, return error. */ 
+                    if(i == argc - 1) { /* last argument, return error. */
                         DISPLAY("Dictionary file expected but not given : %d\n", i);
                         return 1;
                     } else {
@@ -2749,7 +2749,7 @@ int main(int argc, const char** argv)
             } else {
                 result = benchFiles(argv+filenamesStart, argc-filenamesStart, dictFileName, cLevelRun);
             }
-        }   
+        }
     }
 
     if (main_pause) { int unused; printf("press enter...\n"); unused = getchar(); (void)unused; }

From e589ac627673f0241ed95c111bfa0803b3e86ab0 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" 
Date: Mon, 13 Aug 2018 14:57:19 -0700
Subject: [PATCH 193/372] Reformat Introduction Comment and Mention Negative
 Levels

---
 lib/zstd.h | 39 ++++++++++++++++++++++++++-------------
 1 file changed, 26 insertions(+), 13 deletions(-)

diff --git a/lib/zstd.h b/lib/zstd.h
index 0c20bb768..83b760266 100644
--- a/lib/zstd.h
+++ b/lib/zstd.h
@@ -35,26 +35,39 @@ extern "C" {
 #endif
 
 
-/*******************************************************************************************************
+/*******************************************************************************
   Introduction
 
-  zstd, short for Zstandard, is a fast lossless compression algorithm,
-  targeting real-time compression scenarios at zlib-level and better compression ratios.
-  The zstd compression library provides in-memory compression and decompression functions.
-  The library supports compression levels from 1 up to ZSTD_maxCLevel() which is currently 22.
-  Levels >= 20, labeled `--ultra`, should be used with caution, as they require more memory.
+  zstd, short for Zstandard, is a fast lossless compression algorithm, targeting
+  real-time compression scenarios at zlib-level and better compression ratios.
+  The zstd compression library provides in-memory compression and decompression
+  functions.
+
+  The library supports regular compression levels from 1 up to ZSTD_maxCLevel(),
+  which is currently 22. Levels >= 20, labeled `--ultra`, should be used with
+  caution, as they require more memory. The library also offers negative
+  compression levels (all negative integers are valid levels), which extend the
+  range of speed vs. ratio preferences to increasingly extremely strongly
+  prioritize speed.
+
   Compression can be done in:
     - a single step (described as Simple API)
     - a single step, reusing a context (described as Explicit context)
     - unbounded multiple steps (described as Streaming compression)
-  The compression ratio achievable on small data can be highly improved using a dictionary in:
-    - a single step (described as Simple dictionary API)
-    - a single step, reusing a dictionary (described as Bulk-processing dictionary API)
 
-  Advanced experimental functions can be accessed using #define ZSTD_STATIC_LINKING_ONLY before including zstd.h.
-  Advanced experimental APIs shall never be used with a dynamic library.
-  They are not "stable", their definition may change in the future. Only static linking is allowed.
-*********************************************************************************************************/
+  The compression ratio achievable on small data can be highly improved using
+  a dictionary. Dictionary compression can be performed in:
+    - a single step (described as Simple dictionary API)
+    - a single step, reusing a dictionary (described as Bulk-processing
+      dictionary API)
+
+  Advanced experimental functions can be accessed using
+  `#define ZSTD_STATIC_LINKING_ONLY` before including zstd.h.
+
+  Advanced experimental APIs should never be used with a dynamically-linked
+  library. They are not "stable"; their definitions or signatures may change in
+  the future. Only static linking is allowed.
+*******************************************************************************/
 
 /*------   Version   ------*/
 #define ZSTD_VERSION_MAJOR    1

From 8e944ccadebdf845fdd529fcbd49ce9bd0427f99 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" 
Date: Mon, 13 Aug 2018 14:58:57 -0700
Subject: [PATCH 194/372] Regenerate Documentation

---
 doc/zstd_manual.html | 35 ++++++++++++++++++++++++-----------
 1 file changed, 24 insertions(+), 11 deletions(-)

diff --git a/doc/zstd_manual.html b/doc/zstd_manual.html
index bd792008b..8fa6f46cd 100644
--- a/doc/zstd_manual.html
+++ b/doc/zstd_manual.html
@@ -35,22 +35,35 @@
 
 

 
Introduction
-  zstd, short for Zstandard, is a fast lossless compression algorithm,
-  targeting real-time compression scenarios at zlib-level and better compression ratios.
-  The zstd compression library provides in-memory compression and decompression functions.
-  The library supports compression levels from 1 up to ZSTD_maxCLevel() which is currently 22.
-  Levels >= 20, labeled `--ultra`, should be used with caution, as they require more memory.
+  zstd, short for Zstandard, is a fast lossless compression algorithm, targeting
+  real-time compression scenarios at zlib-level and better compression ratios.
+  The zstd compression library provides in-memory compression and decompression
+  functions.
+
+  The library supports regular compression levels from 1 up to ZSTD_maxCLevel(),
+  which is currently 22. Levels >= 20, labeled `--ultra`, should be used with
+  caution, as they require more memory. The library also offers negative
+  compression levels (all negative integers are valid levels), which extend the
+  range of speed vs. ratio preferences to increasingly extremely strongly
+  prioritize speed.
+
   Compression can be done in:
     - a single step (described as Simple API)
     - a single step, reusing a context (described as Explicit context)
     - unbounded multiple steps (described as Streaming compression)
-  The compression ratio achievable on small data can be highly improved using a dictionary in:
-    - a single step (described as Simple dictionary API)
-    - a single step, reusing a dictionary (described as Bulk-processing dictionary API)
 
-  Advanced experimental functions can be accessed using #define ZSTD_STATIC_LINKING_ONLY before including zstd.h.
-  Advanced experimental APIs shall never be used with a dynamic library.
-  They are not "stable", their definition may change in the future. Only static linking is allowed.
+  The compression ratio achievable on small data can be highly improved using
+  a dictionary. Dictionary compression can be performed in:
+    - a single step (described as Simple dictionary API)
+    - a single step, reusing a dictionary (described as Bulk-processing
+      dictionary API)
+
+  Advanced experimental functions can be accessed using
+  `#define ZSTD_STATIC_LINKING_ONLY` before including zstd.h.
+
+  Advanced experimental APIs should never be used with a dynamically-linked
+  library. They are not "stable"; their definitions or signatures may change in
+  the future. Only static linking is allowed.
 

 
 
Version


From 9d6ed9def34fa042291d48637dabc695bfa5d23f Mon Sep 17 00:00:00 2001
From: Jennifer Liu 
Date: Thu, 23 Aug 2018 12:06:20 -0700
Subject: [PATCH 195/372] Merge fastCover into DictBuilder (#1274)

* Minor fix

* Run non-optimize FASTCOVER 5 times in benchmark

* Merge fastCover into dictBuilder

* Fix mixed declaration issue

* Add fastcover to symbol.c

* Add fastCover.c and cover.h to build

* Change fastCover.c to fastcover.c

* Update benchmark to run FASTCOVER in dictBuilder

* Undo spliting fastcover_param into cover_param and f

* Remove convert param functions

* Assign f to parameter

* Add zdict.h to Makefile in lib

* Add cover.h to BUCK

* Cast 1 to U64 before shifting

* Remove trimming of zero freq head and tail in selectSegment and rebenchmark

* Remove f as a separate parameter of tryParam

* Read 8 bytes when d is 6

* Add trimming off zero frequency head and tail

* Use best functions from COVER and remove trimming part(which leads to worse compression ratio after previous bugs were fixed)

* Add finalize= argument to FASTCOVER to specify percentage of training samples passed to ZDICT_finalizeDictionary

* Change nbDmer to always read 8 bytes even when d=6

* Add skip=# argument to allow skipping dmers in computeFrequency in FASTCOVER

* Update comments and benchmarking result

* Change default method of ZDICT_trainFromBuffer to ZDICT_optimizeTrainFromBuffer_fastCover

* Add dictType enum and fix bug about passing zParam when converting to coverParam

* Combine finalize and skip into a single parameter

* Update acceleration parameters and benchmark on 3 sample sets

* Change default splitPoint of FASTCOVER to 0.75 and benchmark first 3 sample sets

* Initialize variables outside of for loop in benchmark.c

* Update benchmark result for hg-manifest

* Remove cover.h from install-includes

* Add explanation of f

* Set default compression level for trainFromBuffer to 3

* Add assertion of fastCoverParams in DiB_trainFromFiles

* Add checkTotalCompressedSize function + some minor fixes

* Add test for multithreading fastCovr

* Initialize segmentFreqs in every FASTCOVER_selectSegment and move mutex_unnlock to end of COVER_best_finish

* Free segmentFreqs

* Initialize segmentFreqs before calling FASTCOVER_buildDictionary instead of in FASTCOVER_selectSegment

* Add FASTCOVER_MEMMULT

* Minor fix

* Update benchmarking result
---
 build/VS2008/fuzzer/fuzzer.vcproj             |   8 +
 build/VS2008/zstd/zstd.vcproj                 |   8 +
 build/VS2008/zstdlib/zstdlib.vcproj           |   8 +
 build/VS2010/fuzzer/fuzzer.vcxproj            |   2 +
 build/VS2010/libzstd-dll/libzstd-dll.vcxproj  |   1 +
 build/VS2010/libzstd/libzstd.vcxproj          |   1 +
 build/VS2010/zstd/zstd.vcxproj                |   2 +
 build/cmake/lib/CMakeLists.txt                |   3 +
 .../benchmarkDictBuilder/Makefile             |  10 +-
 .../benchmarkDictBuilder/README.md            | 925 ++++++++++++++++--
 .../benchmarkDictBuilder/benchmark.c          |  57 +-
 .../fastCover/fastCover.c                     |   4 +-
 lib/BUCK                                      |   1 +
 lib/dictBuilder/cover.c                       | 147 ++-
 lib/dictBuilder/cover.h                       |  83 ++
 lib/dictBuilder/fastcover.c                   | 701 +++++++++++++
 lib/dictBuilder/zdict.c                       |  10 +-
 lib/dictBuilder/zdict.h                       |  59 +-
 programs/README.md                            |   1 +
 programs/dibio.c                              |  50 +-
 programs/dibio.h                              |   2 +-
 programs/zstd.1                               |  21 +-
 programs/zstd.1.md                            |  20 +
 programs/zstdcli.c                            |  76 +-
 tests/playTests.sh                            |  43 +-
 tests/symbols.c                               |   2 +
 26 files changed, 1999 insertions(+), 246 deletions(-)
 create mode 100644 lib/dictBuilder/cover.h
 create mode 100644 lib/dictBuilder/fastcover.c

diff --git a/build/VS2008/fuzzer/fuzzer.vcproj b/build/VS2008/fuzzer/fuzzer.vcproj
index f6ea1f855..4d444caef 100644
--- a/build/VS2008/fuzzer/fuzzer.vcproj
+++ b/build/VS2008/fuzzer/fuzzer.vcproj
@@ -336,6 +336,10 @@
 				RelativePath="..\..\..\lib\dictBuilder\cover.c"
 				>
 			
+			
+			
 			
@@ -482,6 +486,10 @@
 				RelativePath="..\..\..\lib\dictBuilder\zdict.h"
 				>
 			
+			
+			
 			
diff --git a/build/VS2008/zstd/zstd.vcproj b/build/VS2008/zstd/zstd.vcproj
index 5d9f68327..46d7c85dd 100644
--- a/build/VS2008/zstd/zstd.vcproj
+++ b/build/VS2008/zstd/zstd.vcproj
@@ -348,6 +348,10 @@
 				RelativePath="..\..\..\lib\dictBuilder\cover.c"
 				>
 			
+			
+			
 			
@@ -522,6 +526,10 @@
 				RelativePath="..\..\..\lib\dictBuilder\zdict.h"
 				>
 			
+			
+			
 			
diff --git a/build/VS2008/zstdlib/zstdlib.vcproj b/build/VS2008/zstdlib/zstdlib.vcproj
index 7234b02ac..950a6f39c 100644
--- a/build/VS2008/zstdlib/zstdlib.vcproj
+++ b/build/VS2008/zstdlib/zstdlib.vcproj
@@ -332,6 +332,10 @@
 				RelativePath="..\..\..\lib\dictBuilder\cover.c"
 				>
 			
+			
+			
 			
@@ -502,6 +506,10 @@
 				RelativePath="..\..\..\lib\dictBuilder\zdict.h"
 				>
 			
+			
+			
 			
diff --git a/build/VS2010/fuzzer/fuzzer.vcxproj b/build/VS2010/fuzzer/fuzzer.vcxproj
index f0d1ab066..6077cd2c1 100644
--- a/build/VS2010/fuzzer/fuzzer.vcxproj
+++ b/build/VS2010/fuzzer/fuzzer.vcxproj
@@ -176,6 +176,7 @@
     
     
     
+    
     
     
     
@@ -199,6 +200,7 @@
     
     
     
+    
     
     
     
diff --git a/build/VS2010/libzstd-dll/libzstd-dll.vcxproj b/build/VS2010/libzstd-dll/libzstd-dll.vcxproj
index 92d518d3d..6e14e020e 100644
--- a/build/VS2010/libzstd-dll/libzstd-dll.vcxproj
+++ b/build/VS2010/libzstd-dll/libzstd-dll.vcxproj
@@ -43,6 +43,7 @@
     
     
     
+    
     
     
     
diff --git a/build/VS2010/libzstd/libzstd.vcxproj b/build/VS2010/libzstd/libzstd.vcxproj
index c306fcec9..18f5cb533 100644
--- a/build/VS2010/libzstd/libzstd.vcxproj
+++ b/build/VS2010/libzstd/libzstd.vcxproj
@@ -43,6 +43,7 @@
     
     
     
+    
     
     
     
diff --git a/build/VS2010/zstd/zstd.vcxproj b/build/VS2010/zstd/zstd.vcxproj
index 4af281326..936960b58 100644
--- a/build/VS2010/zstd/zstd.vcxproj
+++ b/build/VS2010/zstd/zstd.vcxproj
@@ -40,6 +40,7 @@
     
     
     
+    
     
     
     
@@ -61,6 +62,7 @@
     
     
     
+    
     
     
     
diff --git a/build/cmake/lib/CMakeLists.txt b/build/cmake/lib/CMakeLists.txt
index e84e06301..ffc196ddc 100644
--- a/build/cmake/lib/CMakeLists.txt
+++ b/build/cmake/lib/CMakeLists.txt
@@ -47,6 +47,7 @@ SET(Sources
         ${LIBRARY_DIR}/decompress/huf_decompress.c
         ${LIBRARY_DIR}/decompress/zstd_decompress.c
         ${LIBRARY_DIR}/dictBuilder/cover.c
+        ${LIBRARY_DIR}/dictBuilder/fastcover.c
         ${LIBRARY_DIR}/dictBuilder/divsufsort.c
         ${LIBRARY_DIR}/dictBuilder/zdict.c
         ${LIBRARY_DIR}/deprecated/zbuff_common.c
@@ -74,6 +75,7 @@ SET(Headers
         ${LIBRARY_DIR}/compress/zstd_ldm.h
         ${LIBRARY_DIR}/compress/zstdmt_compress.h
         ${LIBRARY_DIR}/dictBuilder/zdict.h
+        ${LIBRARY_DIR}/dictBuilder/cover.h
         ${LIBRARY_DIR}/deprecated/zbuff.h)
 
 IF (ZSTD_LEGACY_SUPPORT)
@@ -178,6 +180,7 @@ INSTALL(FILES
     ${LIBRARY_DIR}/zstd.h
     ${LIBRARY_DIR}/deprecated/zbuff.h
     ${LIBRARY_DIR}/dictBuilder/zdict.h
+    ${LIBRARY_DIR}/dictBuilder/cover.h
     ${LIBRARY_DIR}/common/zstd_errors.h
     DESTINATION "include")
 
diff --git a/contrib/experimental_dict_builders/benchmarkDictBuilder/Makefile b/contrib/experimental_dict_builders/benchmarkDictBuilder/Makefile
index 681494888..72ce04f2a 100644
--- a/contrib/experimental_dict_builders/benchmarkDictBuilder/Makefile
+++ b/contrib/experimental_dict_builders/benchmarkDictBuilder/Makefile
@@ -2,10 +2,9 @@ ARG :=
 
 CC ?= gcc
 CFLAGS ?= -O3
-INCLUDES := -I ../randomDictBuilder -I ../fastCover -I ../../../programs -I ../../../lib/common -I ../../../lib -I ../../../lib/dictBuilder
+INCLUDES := -I ../randomDictBuilder -I ../../../programs -I ../../../lib/common -I ../../../lib -I ../../../lib/dictBuilder
 
 RANDOM_FILE := ../randomDictBuilder/random.c
-FAST_FILE := ../fastCover/fastCover.c
 IO_FILE := ../randomDictBuilder/io.c
 
 all: run clean
@@ -22,8 +21,8 @@ test: benchmarkTest clean
 benchmarkTest: benchmark test.sh
 	sh test.sh
 
-benchmark: benchmark.o io.o random.o fastCover.o libzstd.a
-	$(CC) $(CFLAGS) benchmark.o io.o random.o fastCover.o libzstd.a -o benchmark
+benchmark: benchmark.o io.o random.o libzstd.a
+	$(CC) $(CFLAGS) benchmark.o io.o random.o libzstd.a -o benchmark
 
 benchmark.o: benchmark.c
 	$(CC) $(CFLAGS) $(INCLUDES) -c benchmark.c
@@ -31,9 +30,6 @@ benchmark.o: benchmark.c
 random.o: $(RANDOM_FILE)
 	$(CC) $(CFLAGS) $(INCLUDES) -c $(RANDOM_FILE)
 
-fastCover.o: $(FAST_FILE)
-	$(CC) $(CFLAGS) $(INCLUDES) -c $(FAST_FILE)
-
 io.o: $(IO_FILE)
 	$(CC) $(CFLAGS) $(INCLUDES) -c $(IO_FILE)
 
diff --git a/contrib/experimental_dict_builders/benchmarkDictBuilder/README.md b/contrib/experimental_dict_builders/benchmarkDictBuilder/README.md
index 559776e2b..6a6c7f1d2 100644
--- a/contrib/experimental_dict_builders/benchmarkDictBuilder/README.md
+++ b/contrib/experimental_dict_builders/benchmarkDictBuilder/README.md
@@ -14,113 +14,836 @@ make ARG="in=../../../lib/dictBuilder in=../../../lib/compress"
 
 ###Benchmarking Result:
 - First Cover is optimize cover, second Cover uses optimized d and k from first one.
-- For every f value of fastCover, the first one is optimize fastCover and the second one uses optimized d and k from first one.
+- For every f value of fastCover, the first one is optimize fastCover and the second one uses optimized d and k from first one. This is run for accel values from 1 to 10.
 - Fourth column is chosen d and fifth column is chosen k
 
 github:
-NODICT       0.000025       2.999642        
-RANDOM       0.030101       8.791189        
-LEGACY       0.913108       8.173529        
-COVER       59.234160       10.652243        8          1298
-COVER       6.258459       10.652243        8          1298
-FAST15       9.959246       10.555630        8          1874
-FAST15       0.077719       10.555630        8          1874
-FAST16       10.028343       10.701698        8          1106
-FAST16       0.078117       10.701698        8          1106
-FAST17       10.567355       10.650652        8          1106
-FAST17       0.124833       10.650652        8          1106
-FAST18       11.795287       10.499142        8          1826
-FAST18       0.086992       10.499142        8          1826
-FAST19       13.132451       10.527140        8          1826
-FAST19       0.134716       10.527140        8          1826
-FAST20       14.366314       10.494710        8          1826
-FAST20       0.128844       10.494710        8          1826
-FAST21       14.941238       10.503488        8          1778
-FAST21       0.134975       10.503488        8          1778
-FAST22       15.146226       10.509284        8          1826
-FAST22       0.146918       10.509284        8          1826
-FAST23       16.260552       10.509284        8          1826
-FAST23       0.158494       10.509284        8          1826
-FAST24       16.806037       10.512369        8          1826
-FAST24       0.190464       10.512369        8          1826
+NODICT       0.000004       2.999642        
+RANDOM       0.024560       8.791189        
+LEGACY       0.727109       8.173529        
+COVER       40.565676       10.652243        8          1298
+COVER       3.608284       10.652243        8          1298
+FAST f=15 a=1       4.181024       10.570882        8          1154
+FAST f=15 a=1       0.040788       10.570882        8          1154
+FAST f=15 a=2       3.548352       10.574287        6          1970
+FAST f=15 a=2       0.035535       10.574287        6          1970
+FAST f=15 a=3       3.287364       10.613950        6          1010
+FAST f=15 a=3       0.032182       10.613950        6          1010
+FAST f=15 a=4       3.184976       10.573883        6          1058
+FAST f=15 a=4       0.029878       10.573883        6          1058
+FAST f=15 a=5       3.045513       10.580640        8          1154
+FAST f=15 a=5       0.022162       10.580640        8          1154
+FAST f=15 a=6       3.003296       10.583677        6          1010
+FAST f=15 a=6       0.028091       10.583677        6          1010
+FAST f=15 a=7       2.952655       10.622551        6          1106
+FAST f=15 a=7       0.02724       10.622551        6          1106
+FAST f=15 a=8       2.945674       10.614657        6          1010
+FAST f=15 a=8       0.027264       10.614657        6          1010
+FAST f=15 a=9       3.153439       10.564018        8          1154
+FAST f=15 a=9       0.020635       10.564018        8          1154
+FAST f=15 a=10       2.950416       10.511454        6          1010
+FAST f=15 a=10       0.026606       10.511454        6          1010
+FAST f=16 a=1       3.970029       10.681035        8          1154
+FAST f=16 a=1       0.038188       10.681035        8          1154
+FAST f=16 a=2       3.422892       10.484978        6          1874
+FAST f=16 a=2       0.034702       10.484978        6          1874
+FAST f=16 a=3       3.215836       10.632631        8          1154
+FAST f=16 a=3       0.026084       10.632631        8          1154
+FAST f=16 a=4       3.081353       10.626533        6          1106
+FAST f=16 a=4       0.030032       10.626533        6          1106
+FAST f=16 a=5       3.041241       10.545027        8          1922
+FAST f=16 a=5       0.022882       10.545027        8          1922
+FAST f=16 a=6       2.989390       10.638284        6          1874
+FAST f=16 a=6       0.028308       10.638284        6          1874
+FAST f=16 a=7       3.001581       10.797136        6          1106
+FAST f=16 a=7       0.027479       10.797136        6          1106
+FAST f=16 a=8       2.984107       10.658356        8          1058
+FAST f=16 a=8       0.021099       10.658356        8          1058
+FAST f=16 a=9       2.925788       10.523869        6          1010
+FAST f=16 a=9       0.026905       10.523869        6          1010
+FAST f=16 a=10       2.889605       10.745841        6          1874
+FAST f=16 a=10       0.026846       10.745841        6          1874
+FAST f=17 a=1       4.031953       10.672080        8          1202
+FAST f=17 a=1       0.040658       10.672080        8          1202
+FAST f=17 a=2       3.458107       10.589352        8          1106
+FAST f=17 a=2       0.02926       10.589352        8          1106
+FAST f=17 a=3       3.291189       10.662714        8          1154
+FAST f=17 a=3       0.026531       10.662714        8          1154
+FAST f=17 a=4       3.154950       10.549456        8          1346
+FAST f=17 a=4       0.024991       10.549456        8          1346
+FAST f=17 a=5       3.092271       10.541670        6          1202
+FAST f=17 a=5       0.038285       10.541670        6          1202
+FAST f=17 a=6       3.166146       10.729112        6          1874
+FAST f=17 a=6       0.038217       10.729112        6          1874
+FAST f=17 a=7       3.035467       10.810485        6          1106
+FAST f=17 a=7       0.036655       10.810485        6          1106
+FAST f=17 a=8       3.035668       10.530532        6          1058
+FAST f=17 a=8       0.037715       10.530532        6          1058
+FAST f=17 a=9       2.987917       10.589802        8          1922
+FAST f=17 a=9       0.02217       10.589802        8          1922
+FAST f=17 a=10       2.981647       10.722579        8          1106
+FAST f=17 a=10       0.021948       10.722579        8          1106
+FAST f=18 a=1       4.067144       10.634943        8          1154
+FAST f=18 a=1       0.041386       10.634943        8          1154
+FAST f=18 a=2       3.507377       10.546230        6          1970
+FAST f=18 a=2       0.037572       10.546230        6          1970
+FAST f=18 a=3       3.323015       10.648061        8          1154
+FAST f=18 a=3       0.028306       10.648061        8          1154
+FAST f=18 a=4       3.216735       10.705402        6          1010
+FAST f=18 a=4       0.030755       10.705402        6          1010
+FAST f=18 a=5       3.175794       10.588154        8          1874
+FAST f=18 a=5       0.025315       10.588154        8          1874
+FAST f=18 a=6       3.127459       10.751104        8          1106
+FAST f=18 a=6       0.023897       10.751104        8          1106
+FAST f=18 a=7       3.083017       10.780402        6          1106
+FAST f=18 a=7       0.029158       10.780402        6          1106
+FAST f=18 a=8       3.069700       10.547226        8          1346
+FAST f=18 a=8       0.024046       10.547226        8          1346
+FAST f=18 a=9       3.056591       10.674759        6          1010
+FAST f=18 a=9       0.028496       10.674759        6          1010
+FAST f=18 a=10       3.063588       10.737578        8          1106
+FAST f=18 a=10       0.023033       10.737578        8          1106
+FAST f=19 a=1       4.164041       10.650333        8          1154
+FAST f=19 a=1       0.042906       10.650333        8          1154
+FAST f=19 a=2       3.585409       10.577066        6          1058
+FAST f=19 a=2       0.038994       10.577066        6          1058
+FAST f=19 a=3       3.439643       10.639403        8          1154
+FAST f=19 a=3       0.028427       10.639403        8          1154
+FAST f=19 a=4       3.268869       10.554410        8          1298
+FAST f=19 a=4       0.026866       10.554410        8          1298
+FAST f=19 a=5       3.238225       10.615109        6          1010
+FAST f=19 a=5       0.03078       10.615109        6          1010
+FAST f=19 a=6       3.199558       10.609782        6          1874
+FAST f=19 a=6       0.030099       10.609782        6          1874
+FAST f=19 a=7       3.132395       10.794753        6          1106
+FAST f=19 a=7       0.028964       10.794753        6          1106
+FAST f=19 a=8       3.148446       10.554842        8          1298
+FAST f=19 a=8       0.024277       10.554842        8          1298
+FAST f=19 a=9       3.108324       10.668763        6          1010
+FAST f=19 a=9       0.02896       10.668763        6          1010
+FAST f=19 a=10       3.159863       10.757347        8          1106
+FAST f=19 a=10       0.023351       10.757347        8          1106
+FAST f=20 a=1       4.462698       10.661788        8          1154
+FAST f=20 a=1       0.047174       10.661788        8          1154
+FAST f=20 a=2       3.820269       10.678612        6          1106
+FAST f=20 a=2       0.040807       10.678612        6          1106
+FAST f=20 a=3       3.644955       10.648424        8          1154
+FAST f=20 a=3       0.031398       10.648424        8          1154
+FAST f=20 a=4       3.546257       10.559756        8          1298
+FAST f=20 a=4       0.029856       10.559756        8          1298
+FAST f=20 a=5       3.485248       10.646637        6          1010
+FAST f=20 a=5       0.033756       10.646637        6          1010
+FAST f=20 a=6       3.490438       10.775824        8          1106
+FAST f=20 a=6       0.028338       10.775824        8          1106
+FAST f=20 a=7       3.631289       10.801795        6          1106
+FAST f=20 a=7       0.035228       10.801795        6          1106
+FAST f=20 a=8       3.758936       10.545116        8          1346
+FAST f=20 a=8       0.027495       10.545116        8          1346
+FAST f=20 a=9       3.707024       10.677454        6          1010
+FAST f=20 a=9       0.031326       10.677454        6          1010
+FAST f=20 a=10       3.586593       10.756017        8          1106
+FAST f=20 a=10       0.027122       10.756017        8          1106
+FAST f=21 a=1       5.701396       10.655398        8          1154
+FAST f=21 a=1       0.067744       10.655398        8          1154
+FAST f=21 a=2       5.270542       10.650743        6          1106
+FAST f=21 a=2       0.052999       10.650743        6          1106
+FAST f=21 a=3       4.945294       10.652380        8          1154
+FAST f=21 a=3       0.052678       10.652380        8          1154
+FAST f=21 a=4       4.894079       10.543185        8          1298
+FAST f=21 a=4       0.04997       10.543185        8          1298
+FAST f=21 a=5       4.785417       10.630321        6          1010
+FAST f=21 a=5       0.045294       10.630321        6          1010
+FAST f=21 a=6       4.789381       10.664477        6          1874
+FAST f=21 a=6       0.046578       10.664477        6          1874
+FAST f=21 a=7       4.302955       10.805179        6          1106
+FAST f=21 a=7       0.041205       10.805179        6          1106
+FAST f=21 a=8       4.034630       10.551211        8          1298
+FAST f=21 a=8       0.040121       10.551211        8          1298
+FAST f=21 a=9       4.523868       10.799114        6          1010
+FAST f=21 a=9       0.043592       10.799114        6          1010
+FAST f=21 a=10       4.760736       10.750255        8          1106
+FAST f=21 a=10       0.043483       10.750255        8          1106
+FAST f=22 a=1       6.743064       10.640537        8          1154
+FAST f=22 a=1       0.086967       10.640537        8          1154
+FAST f=22 a=2       6.121739       10.626638        6          1970
+FAST f=22 a=2       0.066337       10.626638        6          1970
+FAST f=22 a=3       5.248851       10.640688        8          1154
+FAST f=22 a=3       0.054935       10.640688        8          1154
+FAST f=22 a=4       5.436579       10.588333        8          1298
+FAST f=22 a=4       0.064113       10.588333        8          1298
+FAST f=22 a=5       5.812815       10.652653        6          1010
+FAST f=22 a=5       0.058189       10.652653        6          1010
+FAST f=22 a=6       5.745472       10.666437        6          1874
+FAST f=22 a=6       0.057188       10.666437        6          1874
+FAST f=22 a=7       5.716393       10.806911        6          1106
+FAST f=22 a=7       0.056       10.806911        6          1106
+FAST f=22 a=8       5.698799       10.530784        8          1298
+FAST f=22 a=8       0.0583       10.530784        8          1298
+FAST f=22 a=9       5.710533       10.777391        6          1010
+FAST f=22 a=9       0.054945       10.777391        6          1010
+FAST f=22 a=10       5.685395       10.745023        8          1106
+FAST f=22 a=10       0.056526       10.745023        8          1106
+FAST f=23 a=1       7.836923       10.638828        8          1154
+FAST f=23 a=1       0.099522       10.638828        8          1154
+FAST f=23 a=2       6.627834       10.631061        6          1970
+FAST f=23 a=2       0.066769       10.631061        6          1970
+FAST f=23 a=3       5.602533       10.647288        8          1154
+FAST f=23 a=3       0.064513       10.647288        8          1154
+FAST f=23 a=4       6.005580       10.568747        8          1298
+FAST f=23 a=4       0.062022       10.568747        8          1298
+FAST f=23 a=5       5.481816       10.676921        6          1010
+FAST f=23 a=5       0.058959       10.676921        6          1010
+FAST f=23 a=6       5.460444       10.666194        6          1874
+FAST f=23 a=6       0.057687       10.666194        6          1874
+FAST f=23 a=7       5.659822       10.800377        6          1106
+FAST f=23 a=7       0.06783       10.800377        6          1106
+FAST f=23 a=8       6.826940       10.522167        8          1298
+FAST f=23 a=8       0.070533       10.522167        8          1298
+FAST f=23 a=9       6.804757       10.577799        8          1682
+FAST f=23 a=9       0.069949       10.577799        8          1682
+FAST f=23 a=10       6.774933       10.742093        8          1106
+FAST f=23 a=10       0.068395       10.742093        8          1106
+FAST f=24 a=1       8.444110       10.632783        8          1154
+FAST f=24 a=1       0.094357       10.632783        8          1154
+FAST f=24 a=2       7.289578       10.631061        6          1970
+FAST f=24 a=2       0.098515       10.631061        6          1970
+FAST f=24 a=3       8.619780       10.646289        8          1154
+FAST f=24 a=3       0.098041       10.646289        8          1154
+FAST f=24 a=4       8.508455       10.555199        8          1298
+FAST f=24 a=4       0.093885       10.555199        8          1298
+FAST f=24 a=5       8.471145       10.674363        6          1010
+FAST f=24 a=5       0.088676       10.674363        6          1010
+FAST f=24 a=6       8.426727       10.667228        6          1874
+FAST f=24 a=6       0.087247       10.667228        6          1874
+FAST f=24 a=7       8.356826       10.803027        6          1106
+FAST f=24 a=7       0.085835       10.803027        6          1106
+FAST f=24 a=8       6.756811       10.522049        8          1298
+FAST f=24 a=8       0.07107       10.522049        8          1298
+FAST f=24 a=9       6.548169       10.571882        8          1682
+FAST f=24 a=9       0.0713       10.571882        8          1682
+FAST f=24 a=10       8.238079       10.736453        8          1106
+FAST f=24 a=10       0.07004       10.736453        8          1106
+
 
 hg-commands:
-NODICT       0.000026       2.425291        
-RANDOM       0.046270       3.490331        
-LEGACY       0.847904       3.911682        
-COVER       71.691804       4.132653        8          386
-COVER       3.187085       4.132653        8          386
-FAST15       11.593687       3.920720        6          1106
-FAST15       0.082431       3.920720        6          1106
-FAST16       11.775958       4.033306        8          674
-FAST16       0.092587       4.033306        8          674
-FAST17       11.965064       4.064132        8          1490
-FAST17       0.106382       4.064132        8          1490
-FAST18       11.438197       4.086714        8          290
-FAST18       0.097293       4.086714        8          290
-FAST19       12.292512       4.097947        8          578
-FAST19       0.104406       4.097947        8          578
-FAST20       13.857857       4.102851        8          434
-FAST20       0.139467       4.102851        8          434
-FAST21       14.599613       4.105350        8          530
-FAST21       0.189416       4.105350        8          530
-FAST22       15.966109       4.104100        8          530
-FAST22       0.183817       4.104100        8          530
-FAST23       18.033645       4.098110        8          914
-FAST23       0.246641       4.098110        8          914
-FAST24       22.992891       4.117367        8          722
-FAST24       0.285994       4.117367        8          722
+NODICT       0.000005       2.425276        
+RANDOM       0.046332       3.490331        
+LEGACY       0.720351       3.911682        
+COVER       45.507731       4.132653        8          386
+COVER       1.868810       4.132653        8          386
+FAST f=15 a=1       4.561427       3.866894        8          1202
+FAST f=15 a=1       0.048946       3.866894        8          1202
+FAST f=15 a=2       3.574462       3.892119        8          1538
+FAST f=15 a=2       0.033677       3.892119        8          1538
+FAST f=15 a=3       3.230227       3.888791        6          1346
+FAST f=15 a=3       0.034312       3.888791        6          1346
+FAST f=15 a=4       3.042388       3.899739        8          1010
+FAST f=15 a=4       0.024307       3.899739        8          1010
+FAST f=15 a=5       2.800148       3.896220        8          818
+FAST f=15 a=5       0.022331       3.896220        8          818
+FAST f=15 a=6       2.706518       3.882039        8          578
+FAST f=15 a=6       0.020955       3.882039        8          578
+FAST f=15 a=7       2.701820       3.885430        6          866
+FAST f=15 a=7       0.026074       3.885430        6          866
+FAST f=15 a=8       2.604445       3.906932        8          1826
+FAST f=15 a=8       0.021789       3.906932        8          1826
+FAST f=15 a=9       2.598568       3.870324        6          1682
+FAST f=15 a=9       0.026004       3.870324        6          1682
+FAST f=15 a=10       2.575920       3.920783        8          1442
+FAST f=15 a=10       0.020228       3.920783        8          1442
+FAST f=16 a=1       4.630623       4.001430        8          770
+FAST f=16 a=1       0.047497       4.001430        8          770
+FAST f=16 a=2       3.674721       3.974431        8          1874
+FAST f=16 a=2       0.035761       3.974431        8          1874
+FAST f=16 a=3       3.338384       3.978703        8          1010
+FAST f=16 a=3       0.029436       3.978703        8          1010
+FAST f=16 a=4       3.004412       3.983035        8          1010
+FAST f=16 a=4       0.025744       3.983035        8          1010
+FAST f=16 a=5       2.881892       3.987710        8          770
+FAST f=16 a=5       0.023211       3.987710        8          770
+FAST f=16 a=6       2.807410       3.952717        8          1298
+FAST f=16 a=6       0.023199       3.952717        8          1298
+FAST f=16 a=7       2.819623       3.994627        8          770
+FAST f=16 a=7       0.021806       3.994627        8          770
+FAST f=16 a=8       2.740092       3.954032        8          1826
+FAST f=16 a=8       0.0226       3.954032        8          1826
+FAST f=16 a=9       2.682564       3.969879        6          1442
+FAST f=16 a=9       0.026324       3.969879        6          1442
+FAST f=16 a=10       2.657959       3.969755        8          674
+FAST f=16 a=10       0.020413       3.969755        8          674
+FAST f=17 a=1       4.729228       4.046000        8          530
+FAST f=17 a=1       0.049703       4.046000        8          530
+FAST f=17 a=2       3.764510       3.991519        8          1970
+FAST f=17 a=2       0.038195       3.991519        8          1970
+FAST f=17 a=3       3.416992       4.006296        6          914
+FAST f=17 a=3       0.036244       4.006296        6          914
+FAST f=17 a=4       3.145626       3.979182        8          1970
+FAST f=17 a=4       0.028676       3.979182        8          1970
+FAST f=17 a=5       2.995070       4.050070        8          770
+FAST f=17 a=5       0.025707       4.050070        8          770
+FAST f=17 a=6       2.911833       4.040024        8          770
+FAST f=17 a=6       0.02453       4.040024        8          770
+FAST f=17 a=7       2.894796       4.015884        8          818
+FAST f=17 a=7       0.023956       4.015884        8          818
+FAST f=17 a=8       2.789962       4.039303        8          530
+FAST f=17 a=8       0.023219       4.039303        8          530
+FAST f=17 a=9       2.787625       3.996762        8          1634
+FAST f=17 a=9       0.023651       3.996762        8          1634
+FAST f=17 a=10       2.754796       4.005059        8          1058
+FAST f=17 a=10       0.022537       4.005059        8          1058
+FAST f=18 a=1       4.779117       4.038214        8          242
+FAST f=18 a=1       0.048814       4.038214        8          242
+FAST f=18 a=2       3.829753       4.045768        8          722
+FAST f=18 a=2       0.036541       4.045768        8          722
+FAST f=18 a=3       3.495053       4.021497        8          770
+FAST f=18 a=3       0.032648       4.021497        8          770
+FAST f=18 a=4       3.221395       4.039623        8          770
+FAST f=18 a=4       0.027818       4.039623        8          770
+FAST f=18 a=5       3.059369       4.050414        8          530
+FAST f=18 a=5       0.026296       4.050414        8          530
+FAST f=18 a=6       3.019292       4.010714        6          962
+FAST f=18 a=6       0.031104       4.010714        6          962
+FAST f=18 a=7       2.949322       4.031439        6          770
+FAST f=18 a=7       0.030745       4.031439        6          770
+FAST f=18 a=8       2.876425       4.032088        6          386
+FAST f=18 a=8       0.027407       4.032088        6          386
+FAST f=18 a=9       2.850958       4.053372        8          674
+FAST f=18 a=9       0.023799       4.053372        8          674
+FAST f=18 a=10       2.884352       4.020148        8          1730
+FAST f=18 a=10       0.024401       4.020148        8          1730
+FAST f=19 a=1       4.815669       4.061203        8          674
+FAST f=19 a=1       0.051425       4.061203        8          674
+FAST f=19 a=2       3.951356       4.013822        8          1442
+FAST f=19 a=2       0.039968       4.013822        8          1442
+FAST f=19 a=3       3.554682       4.050425        8          722
+FAST f=19 a=3       0.032725       4.050425        8          722
+FAST f=19 a=4       3.242585       4.054677        8          722
+FAST f=19 a=4       0.028194       4.054677        8          722
+FAST f=19 a=5       3.105909       4.064524        8          818
+FAST f=19 a=5       0.02675       4.064524        8          818
+FAST f=19 a=6       3.059901       4.036857        8          1250
+FAST f=19 a=6       0.026396       4.036857        8          1250
+FAST f=19 a=7       3.016151       4.068234        6          770
+FAST f=19 a=7       0.031501       4.068234        6          770
+FAST f=19 a=8       2.962902       4.077509        8          530
+FAST f=19 a=8       0.023333       4.077509        8          530
+FAST f=19 a=9       2.899607       4.067328        8          530
+FAST f=19 a=9       0.024553       4.067328        8          530
+FAST f=19 a=10       2.950978       4.059901        8          434
+FAST f=19 a=10       0.023852       4.059901        8          434
+FAST f=20 a=1       5.259834       4.027579        8          1634
+FAST f=20 a=1       0.061123       4.027579        8          1634
+FAST f=20 a=2       4.382150       4.025093        8          1634
+FAST f=20 a=2       0.048009       4.025093        8          1634
+FAST f=20 a=3       4.104323       4.060842        8          530
+FAST f=20 a=3       0.040965       4.060842        8          530
+FAST f=20 a=4       3.853340       4.023504        6          914
+FAST f=20 a=4       0.041072       4.023504        6          914
+FAST f=20 a=5       3.728841       4.018089        6          1634
+FAST f=20 a=5       0.037469       4.018089        6          1634
+FAST f=20 a=6       3.683045       4.069138        8          578
+FAST f=20 a=6       0.028011       4.069138        8          578
+FAST f=20 a=7       3.726973       4.063160        8          722
+FAST f=20 a=7       0.028437       4.063160        8          722
+FAST f=20 a=8       3.555073       4.057690        8          386
+FAST f=20 a=8       0.027588       4.057690        8          386
+FAST f=20 a=9       3.551095       4.067253        8          482
+FAST f=20 a=9       0.025976       4.067253        8          482
+FAST f=20 a=10       3.490127       4.068518        8          530
+FAST f=20 a=10       0.025971       4.068518        8          530
+FAST f=21 a=1       7.343816       4.064945        8          770
+FAST f=21 a=1       0.085035       4.064945        8          770
+FAST f=21 a=2       5.930894       4.048206        8          386
+FAST f=21 a=2       0.067349       4.048206        8          386
+FAST f=21 a=3       6.770775       4.063417        8          578
+FAST f=21 a=3       0.077104       4.063417        8          578
+FAST f=21 a=4       6.889409       4.066761        8          626
+FAST f=21 a=4       0.0717       4.066761        8          626
+FAST f=21 a=5       6.714896       4.051813        8          914
+FAST f=21 a=5       0.071026       4.051813        8          914
+FAST f=21 a=6       6.539890       4.047263        8          1922
+FAST f=21 a=6       0.07127       4.047263        8          1922
+FAST f=21 a=7       6.511052       4.068373        8          482
+FAST f=21 a=7       0.065467       4.068373        8          482
+FAST f=21 a=8       6.458788       4.071597        8          482
+FAST f=21 a=8       0.063817       4.071597        8          482
+FAST f=21 a=9       6.377591       4.052905        8          434
+FAST f=21 a=9       0.063112       4.052905        8          434
+FAST f=21 a=10       6.360752       4.047773        8          530
+FAST f=21 a=10       0.063606       4.047773        8          530
+FAST f=22 a=1       10.523471       4.040812        8          962
+FAST f=22 a=1       0.14214       4.040812        8          962
+FAST f=22 a=2       9.454758       4.059396        8          914
+FAST f=22 a=2       0.118343       4.059396        8          914
+FAST f=22 a=3       9.043197       4.043019        8          1922
+FAST f=22 a=3       0.109798       4.043019        8          1922
+FAST f=22 a=4       8.716261       4.044819        8          770
+FAST f=22 a=4       0.099687       4.044819        8          770
+FAST f=22 a=5       8.529472       4.070576        8          530
+FAST f=22 a=5       0.093127       4.070576        8          530
+FAST f=22 a=6       8.424241       4.070565        8          722
+FAST f=22 a=6       0.093703       4.070565        8          722
+FAST f=22 a=7       8.403391       4.070591        8          578
+FAST f=22 a=7       0.089763       4.070591        8          578
+FAST f=22 a=8       8.285221       4.089171        8          530
+FAST f=22 a=8       0.087716       4.089171        8          530
+FAST f=22 a=9       8.282506       4.047470        8          722
+FAST f=22 a=9       0.089773       4.047470        8          722
+FAST f=22 a=10       8.241809       4.064151        8          818
+FAST f=22 a=10       0.090413       4.064151        8          818
+FAST f=23 a=1       12.389208       4.051635        6          530
+FAST f=23 a=1       0.147796       4.051635        6          530
+FAST f=23 a=2       11.300910       4.042835        6          914
+FAST f=23 a=2       0.133178       4.042835        6          914
+FAST f=23 a=3       10.879455       4.047415        8          626
+FAST f=23 a=3       0.129571       4.047415        8          626
+FAST f=23 a=4       10.522718       4.038269        6          914
+FAST f=23 a=4       0.118121       4.038269        6          914
+FAST f=23 a=5       10.348043       4.066884        8          434
+FAST f=23 a=5       0.112098       4.066884        8          434
+FAST f=23 a=6       10.238630       4.048635        8          1010
+FAST f=23 a=6       0.120281       4.048635        8          1010
+FAST f=23 a=7       10.213255       4.061809        8          530
+FAST f=23 a=7       0.1121       4.061809        8          530
+FAST f=23 a=8       10.107879       4.074104        8          818
+FAST f=23 a=8       0.116544       4.074104        8          818
+FAST f=23 a=9       10.063424       4.064811        8          674
+FAST f=23 a=9       0.109045       4.064811        8          674
+FAST f=23 a=10       10.035801       4.054918        8          530
+FAST f=23 a=10       0.108735       4.054918        8          530
+FAST f=24 a=1       14.963878       4.073490        8          722
+FAST f=24 a=1       0.206344       4.073490        8          722
+FAST f=24 a=2       13.833472       4.036100        8          962
+FAST f=24 a=2       0.17486       4.036100        8          962
+FAST f=24 a=3       13.404631       4.026281        6          1106
+FAST f=24 a=3       0.153961       4.026281        6          1106
+FAST f=24 a=4       13.041164       4.065448        8          674
+FAST f=24 a=4       0.155509       4.065448        8          674
+FAST f=24 a=5       12.879412       4.054636        8          674
+FAST f=24 a=5       0.148282       4.054636        8          674
+FAST f=24 a=6       12.773736       4.081376        8          530
+FAST f=24 a=6       0.142563       4.081376        8          530
+FAST f=24 a=7       12.711310       4.059834        8          770
+FAST f=24 a=7       0.149321       4.059834        8          770
+FAST f=24 a=8       12.635459       4.052050        8          1298
+FAST f=24 a=8       0.15095       4.052050        8          1298
+FAST f=24 a=9       12.558104       4.076516        8          722
+FAST f=24 a=9       0.144361       4.076516        8          722
+FAST f=24 a=10       10.661348       4.062137        8          818
+FAST f=24 a=10       0.108232       4.062137        8          818
+
 
 hg-changelog:
-NODICT       0.000007       1.377613        
-RANDOM       0.297345       2.097487        
-LEGACY       2.633992       2.058907        
-COVER       219.179786       2.189685        8          98
-COVER       6.620852       2.189685        8          98
-FAST15       47.635082       2.130794        6          386
-FAST15       0.321297       2.130794        6          386
-FAST16       43.837676       2.144845        8          194
-FAST16       0.312640       2.144845        8          194
-FAST17       49.349017       2.156099        8          242
-FAST17       0.348459       2.156099        8          242
-FAST18       51.153784       2.172439        6          98
-FAST18       0.353106       2.172439        6          98
-FAST19       52.627045       2.180321        6          98
-FAST19       0.390612       2.180321        6          98
-FAST20       63.748782       2.187431        6          98
-FAST20       0.489544       2.187431        6          98
-FAST21       68.709198       2.184185        6          146
-FAST21       0.530852       2.184185        6          146
-FAST22       68.491639       2.182830        6          98
-FAST22       0.645699       2.182830        6          98
-FAST23       72.558688       2.186399        8          98
-FAST23       0.593539       2.186399        8          98
-FAST24       76.137195       2.185608        6          98
-FAST24       0.680132       2.185608        6          98
+NODICT       0.000017       1.377590        
+RANDOM       0.186171       2.097487        
+LEGACY       1.670867       2.058907        
+COVER       173.561948       2.189685        8          98
+COVER       4.811180       2.189685        8          98
+FAST f=15 a=1       18.685906       2.129682        8          434
+FAST f=15 a=1       0.173376       2.129682        8          434
+FAST f=15 a=2       12.928259       2.131890        8          482
+FAST f=15 a=2       0.102582       2.131890        8          482
+FAST f=15 a=3       11.132343       2.128027        8          386
+FAST f=15 a=3       0.077122       2.128027        8          386
+FAST f=15 a=4       10.120683       2.125797        8          434
+FAST f=15 a=4       0.065175       2.125797        8          434
+FAST f=15 a=5       9.479092       2.127697        8          386
+FAST f=15 a=5       0.057905       2.127697        8          386
+FAST f=15 a=6       9.159523       2.127132        8          1682
+FAST f=15 a=6       0.058604       2.127132        8          1682
+FAST f=15 a=7       8.724003       2.129914        8          434
+FAST f=15 a=7       0.0493       2.129914        8          434
+FAST f=15 a=8       8.595001       2.127137        8          338
+FAST f=15 a=8       0.0474       2.127137        8          338
+FAST f=15 a=9       8.356405       2.125512        8          482
+FAST f=15 a=9       0.046126       2.125512        8          482
+FAST f=15 a=10       8.207111       2.126066        8          338
+FAST f=15 a=10       0.043292       2.126066        8          338
+FAST f=16 a=1       18.464436       2.144040        8          242
+FAST f=16 a=1       0.172156       2.144040        8          242
+FAST f=16 a=2       12.844825       2.148171        8          194
+FAST f=16 a=2       0.099619       2.148171        8          194
+FAST f=16 a=3       11.082568       2.140837        8          290
+FAST f=16 a=3       0.079165       2.140837        8          290
+FAST f=16 a=4       10.066749       2.144405        8          386
+FAST f=16 a=4       0.068411       2.144405        8          386
+FAST f=16 a=5       9.501121       2.140720        8          386
+FAST f=16 a=5       0.061316       2.140720        8          386
+FAST f=16 a=6       9.179332       2.139478        8          386
+FAST f=16 a=6       0.056322       2.139478        8          386
+FAST f=16 a=7       8.849438       2.142412        8          194
+FAST f=16 a=7       0.050493       2.142412        8          194
+FAST f=16 a=8       8.810919       2.143454        8          434
+FAST f=16 a=8       0.051304       2.143454        8          434
+FAST f=16 a=9       8.553900       2.140339        8          194
+FAST f=16 a=9       0.047285       2.140339        8          194
+FAST f=16 a=10       8.398027       2.143130        8          386
+FAST f=16 a=10       0.046386       2.143130        8          386
+FAST f=17 a=1       18.644657       2.157192        8          98
+FAST f=17 a=1       0.173884       2.157192        8          98
+FAST f=17 a=2       13.071242       2.159830        8          146
+FAST f=17 a=2       0.10388       2.159830        8          146
+FAST f=17 a=3       11.332366       2.153654        6          194
+FAST f=17 a=3       0.08983       2.153654        6          194
+FAST f=17 a=4       10.362413       2.156813        8          242
+FAST f=17 a=4       0.070389       2.156813        8          242
+FAST f=17 a=5       9.808159       2.155098        6          338
+FAST f=17 a=5       0.072661       2.155098        6          338
+FAST f=17 a=6       9.451165       2.153845        6          146
+FAST f=17 a=6       0.064959       2.153845        6          146
+FAST f=17 a=7       9.163097       2.155424        6          242
+FAST f=17 a=7       0.064323       2.155424        6          242
+FAST f=17 a=8       9.047276       2.156640        8          242
+FAST f=17 a=8       0.053382       2.156640        8          242
+FAST f=17 a=9       8.807671       2.152396        8          146
+FAST f=17 a=9       0.049617       2.152396        8          146
+FAST f=17 a=10       8.649827       2.152370        8          146
+FAST f=17 a=10       0.047849       2.152370        8          146
+FAST f=18 a=1       18.809502       2.168116        8          98
+FAST f=18 a=1       0.175226       2.168116        8          98
+FAST f=18 a=2       13.756502       2.170870        6          242
+FAST f=18 a=2       0.119507       2.170870        6          242
+FAST f=18 a=3       12.059748       2.163094        6          98
+FAST f=18 a=3       0.093912       2.163094        6          98
+FAST f=18 a=4       11.410294       2.172372        8          98
+FAST f=18 a=4       0.073048       2.172372        8          98
+FAST f=18 a=5       10.560297       2.166388        8          98
+FAST f=18 a=5       0.065136       2.166388        8          98
+FAST f=18 a=6       10.071390       2.162672        8          98
+FAST f=18 a=6       0.059402       2.162672        8          98
+FAST f=18 a=7       10.084214       2.166624        6          194
+FAST f=18 a=7       0.073276       2.166624        6          194
+FAST f=18 a=8       9.953226       2.167454        8          98
+FAST f=18 a=8       0.053659       2.167454        8          98
+FAST f=18 a=9       8.982461       2.161593        6          146
+FAST f=18 a=9       0.05955       2.161593        6          146
+FAST f=18 a=10       8.986092       2.164373        6          242
+FAST f=18 a=10       0.059135       2.164373        6          242
+FAST f=19 a=1       18.908277       2.176021        8          98
+FAST f=19 a=1       0.177316       2.176021        8          98
+FAST f=19 a=2       13.471313       2.176103        8          98
+FAST f=19 a=2       0.106344       2.176103        8          98
+FAST f=19 a=3       11.571406       2.172812        8          98
+FAST f=19 a=3       0.083293       2.172812        8          98
+FAST f=19 a=4       10.632775       2.177770        6          146
+FAST f=19 a=4       0.079864       2.177770        6          146
+FAST f=19 a=5       10.030190       2.175574        6          146
+FAST f=19 a=5       0.07223       2.175574        6          146
+FAST f=19 a=6       9.717818       2.169997        8          98
+FAST f=19 a=6       0.060049       2.169997        8          98
+FAST f=19 a=7       9.397531       2.172770        8          146
+FAST f=19 a=7       0.057188       2.172770        8          146
+FAST f=19 a=8       9.281061       2.175822        8          98
+FAST f=19 a=8       0.053711       2.175822        8          98
+FAST f=19 a=9       9.165242       2.169849        6          146
+FAST f=19 a=9       0.059898       2.169849        6          146
+FAST f=19 a=10       9.048763       2.173394        8          98
+FAST f=19 a=10       0.049757       2.173394        8          98
+FAST f=20 a=1       21.166917       2.183923        6          98
+FAST f=20 a=1       0.205425       2.183923        6          98
+FAST f=20 a=2       15.642753       2.182349        6          98
+FAST f=20 a=2       0.135957       2.182349        6          98
+FAST f=20 a=3       14.053730       2.173544        6          98
+FAST f=20 a=3       0.11266       2.173544        6          98
+FAST f=20 a=4       15.270019       2.183656        8          98
+FAST f=20 a=4       0.107892       2.183656        8          98
+FAST f=20 a=5       15.497927       2.174661        6          98
+FAST f=20 a=5       0.100305       2.174661        6          98
+FAST f=20 a=6       13.973505       2.172391        8          98
+FAST f=20 a=6       0.087565       2.172391        8          98
+FAST f=20 a=7       14.083296       2.172443        8          98
+FAST f=20 a=7       0.078062       2.172443        8          98
+FAST f=20 a=8       12.560048       2.175581        8          98
+FAST f=20 a=8       0.070282       2.175581        8          98
+FAST f=20 a=9       13.078645       2.173975        6          146
+FAST f=20 a=9       0.081041       2.173975        6          146
+FAST f=20 a=10       12.823328       2.177778        8          98
+FAST f=20 a=10       0.074522       2.177778        8          98
+FAST f=21 a=1       29.825370       2.183057        6          98
+FAST f=21 a=1       0.334453       2.183057        6          98
+FAST f=21 a=2       29.476474       2.182752        8          98
+FAST f=21 a=2       0.286602       2.182752        8          98
+FAST f=21 a=3       25.937186       2.175867        8          98
+FAST f=21 a=3       0.17626       2.175867        8          98
+FAST f=21 a=4       20.413865       2.179780        8          98
+FAST f=21 a=4       0.206085       2.179780        8          98
+FAST f=21 a=5       20.541889       2.178328        6          146
+FAST f=21 a=5       0.199157       2.178328        6          146
+FAST f=21 a=6       21.090670       2.174443        6          146
+FAST f=21 a=6       0.190645       2.174443        6          146
+FAST f=21 a=7       20.221569       2.177384        6          146
+FAST f=21 a=7       0.184278       2.177384        6          146
+FAST f=21 a=8       20.322357       2.179456        6          98
+FAST f=21 a=8       0.178458       2.179456        6          98
+FAST f=21 a=9       20.683912       2.174396        6          146
+FAST f=21 a=9       0.190829       2.174396        6          146
+FAST f=21 a=10       20.840865       2.174905        8          98
+FAST f=21 a=10       0.172515       2.174905        8          98
+FAST f=22 a=1       36.822827       2.181612        6          98
+FAST f=22 a=1       0.437389       2.181612        6          98
+FAST f=22 a=2       30.616902       2.183142        8          98
+FAST f=22 a=2       0.324284       2.183142        8          98
+FAST f=22 a=3       28.472482       2.178130        8          98
+FAST f=22 a=3       0.236538       2.178130        8          98
+FAST f=22 a=4       25.847028       2.181878        8          98
+FAST f=22 a=4       0.263744       2.181878        8          98
+FAST f=22 a=5       27.095881       2.180775        8          98
+FAST f=22 a=5       0.24988       2.180775        8          98
+FAST f=22 a=6       25.939172       2.170916        8          98
+FAST f=22 a=6       0.240033       2.170916        8          98
+FAST f=22 a=7       27.064194       2.177849        8          98
+FAST f=22 a=7       0.242383       2.177849        8          98
+FAST f=22 a=8       25.140221       2.178216        8          98
+FAST f=22 a=8       0.237601       2.178216        8          98
+FAST f=22 a=9       25.505283       2.177455        6          146
+FAST f=22 a=9       0.223217       2.177455        6          146
+FAST f=22 a=10       24.529362       2.176705        6          98
+FAST f=22 a=10       0.222876       2.176705        6          98
+FAST f=23 a=1       39.127310       2.183006        6          98
+FAST f=23 a=1       0.417338       2.183006        6          98
+FAST f=23 a=2       32.468161       2.183524        6          98
+FAST f=23 a=2       0.351645       2.183524        6          98
+FAST f=23 a=3       31.577620       2.172604        6          98
+FAST f=23 a=3       0.319659       2.172604        6          98
+FAST f=23 a=4       30.129247       2.183932        6          98
+FAST f=23 a=4       0.307239       2.183932        6          98
+FAST f=23 a=5       29.103376       2.183529        6          146
+FAST f=23 a=5       0.285533       2.183529        6          146
+FAST f=23 a=6       29.776045       2.174367        8          98
+FAST f=23 a=6       0.276846       2.174367        8          98
+FAST f=23 a=7       28.940407       2.178022        6          146
+FAST f=23 a=7       0.274082       2.178022        6          146
+FAST f=23 a=8       29.256009       2.179462        6          98
+FAST f=23 a=8       0.26949       2.179462        6          98
+FAST f=23 a=9       29.347312       2.170407        8          98
+FAST f=23 a=9       0.265034       2.170407        8          98
+FAST f=23 a=10       29.140081       2.171762        8          98
+FAST f=23 a=10       0.259183       2.171762        8          98
+FAST f=24 a=1       44.871179       2.182115        6          98
+FAST f=24 a=1       0.509433       2.182115        6          98
+FAST f=24 a=2       38.694867       2.180549        8          98
+FAST f=24 a=2       0.406695       2.180549        8          98
+FAST f=24 a=3       38.363769       2.172821        8          98
+FAST f=24 a=3       0.359581       2.172821        8          98
+FAST f=24 a=4       36.580797       2.184142        8          98
+FAST f=24 a=4       0.340614       2.184142        8          98
+FAST f=24 a=5       33.125701       2.183301        8          98
+FAST f=24 a=5       0.324874       2.183301        8          98
+FAST f=24 a=6       34.776068       2.173019        6          146
+FAST f=24 a=6       0.340397       2.173019        6          146
+FAST f=24 a=7       34.417625       2.176561        6          146
+FAST f=24 a=7       0.308223       2.176561        6          146
+FAST f=24 a=8       35.470291       2.182161        6          98
+FAST f=24 a=8       0.307724       2.182161        6          98
+FAST f=24 a=9       34.927252       2.172682        6          146
+FAST f=24 a=9       0.300598       2.172682        6          146
+FAST f=24 a=10       33.238355       2.173395        6          98
+FAST f=24 a=10       0.249916       2.173395        6          98
+
 
 hg-manifest:
-NODICT       0.000026       1.866385        
-RANDOM       0.784554       2.309436        
-LEGACY       10.193714       2.506977        
-COVER       988.206583       2.582528        8          434
-COVER       39.726199       2.582528        8          434
-FAST15       168.388819       2.392920        6          1826
-FAST15       1.272178       2.392920        6          1826
-FAST16       161.822607       2.480762        6          1922
-FAST16       1.164908       2.480762        6          1922
-FAST17       157.688544       2.548285        6          1682
-FAST17       1.222439       2.548285        6          1682
-FAST18       154.529585       2.567634        6          386
-FAST18       1.217596       2.567634        6          386
-FAST19       160.244979       2.581653        8          338
-FAST19       1.282450       2.581653        8          338
-FAST20       191.503297       2.586881        8          194
-FAST20       2.009748       2.586881        8          194
-FAST21       226.389709       2.590051        6          242
-FAST21       2.494543       2.590051        6          242
-FAST22       217.859055       2.591376        6          194
-FAST22       2.295693       2.591376        6          194
-FAST23       236.819791       2.591131        8          434
-FAST23       2.744711       2.591131        8          434
-FAST24       269.187800       2.591548        6          290
-FAST24       2.923671       2.591548        6          290
+NODICT       0.000004       1.866377        
+RANDOM       0.696346       2.309436        
+LEGACY       7.064527       2.506977        
+COVER       876.312865       2.582528        8          434
+COVER       35.684533       2.582528        8          434
+FAST f=15 a=1       76.618201       2.404013        8          1202
+FAST f=15 a=1       0.700722       2.404013        8          1202
+FAST f=15 a=2       49.213058       2.409248        6          1826
+FAST f=15 a=2       0.473393       2.409248        6          1826
+FAST f=15 a=3       41.753197       2.409677        8          1490
+FAST f=15 a=3       0.336848       2.409677        8          1490
+FAST f=15 a=4       38.648295       2.407996        8          1538
+FAST f=15 a=4       0.283952       2.407996        8          1538
+FAST f=15 a=5       36.144936       2.402895        8          1874
+FAST f=15 a=5       0.270128       2.402895        8          1874
+FAST f=15 a=6       35.484675       2.394873        8          1586
+FAST f=15 a=6       0.251637       2.394873        8          1586
+FAST f=15 a=7       34.280599       2.397311        8          1778
+FAST f=15 a=7       0.23984       2.397311        8          1778
+FAST f=15 a=8       32.122572       2.396089        6          1490
+FAST f=15 a=8       0.251508       2.396089        6          1490
+FAST f=15 a=9       29.909842       2.390092        6          1970
+FAST f=15 a=9       0.251233       2.390092        6          1970
+FAST f=15 a=10       30.102938       2.400086        6          1682
+FAST f=15 a=10       0.23688       2.400086        6          1682
+FAST f=16 a=1       67.750401       2.475460        6          1346
+FAST f=16 a=1       0.796035       2.475460        6          1346
+FAST f=16 a=2       52.812027       2.480860        6          1730
+FAST f=16 a=2       0.480384       2.480860        6          1730
+FAST f=16 a=3       44.179259       2.469304        8          1970
+FAST f=16 a=3       0.332657       2.469304        8          1970
+FAST f=16 a=4       37.612728       2.478208        6          1970
+FAST f=16 a=4       0.32498       2.478208        6          1970
+FAST f=16 a=5       35.056222       2.475568        6          1298
+FAST f=16 a=5       0.302824       2.475568        6          1298
+FAST f=16 a=6       34.713012       2.486079        8          1730
+FAST f=16 a=6       0.24755       2.486079        8          1730
+FAST f=16 a=7       33.713687       2.477180        6          1682
+FAST f=16 a=7       0.280358       2.477180        6          1682
+FAST f=16 a=8       31.571412       2.475418        8          1538
+FAST f=16 a=8       0.241241       2.475418        8          1538
+FAST f=16 a=9       31.608069       2.478263        8          1922
+FAST f=16 a=9       0.241764       2.478263        8          1922
+FAST f=16 a=10       31.358002       2.472263        8          1442
+FAST f=16 a=10       0.221661       2.472263        8          1442
+FAST f=17 a=1       66.185775       2.536085        6          1346
+FAST f=17 a=1       0.713549       2.536085        6          1346
+FAST f=17 a=2       50.365000       2.546105        8          1298
+FAST f=17 a=2       0.467846       2.546105        8          1298
+FAST f=17 a=3       42.712843       2.536250        8          1298
+FAST f=17 a=3       0.34047       2.536250        8          1298
+FAST f=17 a=4       39.514227       2.535555        8          1442
+FAST f=17 a=4       0.302989       2.535555        8          1442
+FAST f=17 a=5       35.189292       2.524925        8          1202
+FAST f=17 a=5       0.273451       2.524925        8          1202
+FAST f=17 a=6       35.791683       2.523466        8          1202
+FAST f=17 a=6       0.268261       2.523466        8          1202
+FAST f=17 a=7       37.416136       2.526625        6          1010
+FAST f=17 a=7       0.277558       2.526625        6          1010
+FAST f=17 a=8       37.084707       2.533274        6          1250
+FAST f=17 a=8       0.285104       2.533274        6          1250
+FAST f=17 a=9       34.183814       2.532765        8          1298
+FAST f=17 a=9       0.235133       2.532765        8          1298
+FAST f=17 a=10       31.149235       2.528722        8          1346
+FAST f=17 a=10       0.232679       2.528722        8          1346
+FAST f=18 a=1       72.942176       2.559857        6          386
+FAST f=18 a=1       0.718618       2.559857        6          386
+FAST f=18 a=2       51.690440       2.559572        8          290
+FAST f=18 a=2       0.403978       2.559572        8          290
+FAST f=18 a=3       45.344908       2.561040        8          962
+FAST f=18 a=3       0.357205       2.561040        8          962
+FAST f=18 a=4       39.804522       2.558446        8          1010
+FAST f=18 a=4       0.310526       2.558446        8          1010
+FAST f=18 a=5       38.134888       2.561811        8          626
+FAST f=18 a=5       0.273743       2.561811        8          626
+FAST f=18 a=6       35.091890       2.555518        8          722
+FAST f=18 a=6       0.260135       2.555518        8          722
+FAST f=18 a=7       34.639523       2.562938        8          290
+FAST f=18 a=7       0.234294       2.562938        8          290
+FAST f=18 a=8       36.076431       2.563567        8          1586
+FAST f=18 a=8       0.274075       2.563567        8          1586
+FAST f=18 a=9       36.376433       2.560950        8          722
+FAST f=18 a=9       0.240106       2.560950        8          722
+FAST f=18 a=10       32.624790       2.559340        8          578
+FAST f=18 a=10       0.234704       2.559340        8          578
+FAST f=19 a=1       70.513761       2.572441        8          194
+FAST f=19 a=1       0.726112       2.572441        8          194
+FAST f=19 a=2       59.263032       2.574560        8          482
+FAST f=19 a=2       0.451554       2.574560        8          482
+FAST f=19 a=3       51.509594       2.571546        6          194
+FAST f=19 a=3       0.393014       2.571546        6          194
+FAST f=19 a=4       55.393906       2.573386        8          482
+FAST f=19 a=4       0.38819       2.573386        8          482
+FAST f=19 a=5       43.201736       2.567589        8          674
+FAST f=19 a=5       0.292155       2.567589        8          674
+FAST f=19 a=6       42.911687       2.572666        6          434
+FAST f=19 a=6       0.303988       2.572666        6          434
+FAST f=19 a=7       44.687591       2.573613        6          290
+FAST f=19 a=7       0.308721       2.573613        6          290
+FAST f=19 a=8       37.372868       2.571039        6          194
+FAST f=19 a=8       0.287137       2.571039        6          194
+FAST f=19 a=9       36.074230       2.566473        6          482
+FAST f=19 a=9       0.280721       2.566473        6          482
+FAST f=19 a=10       33.731720       2.570306        8          194
+FAST f=19 a=10       0.224073       2.570306        8          194
+FAST f=20 a=1       79.670634       2.581146        6          290
+FAST f=20 a=1       0.899986       2.581146        6          290
+FAST f=20 a=2       58.827141       2.579782        8          386
+FAST f=20 a=2       0.602288       2.579782        8          386
+FAST f=20 a=3       51.289004       2.579627        8          722
+FAST f=20 a=3       0.446091       2.579627        8          722
+FAST f=20 a=4       47.711068       2.581508        8          722
+FAST f=20 a=4       0.473007       2.581508        8          722
+FAST f=20 a=5       47.402929       2.578062        6          434
+FAST f=20 a=5       0.497131       2.578062        6          434
+FAST f=20 a=6       54.797102       2.577365        8          482
+FAST f=20 a=6       0.515061       2.577365        8          482
+FAST f=20 a=7       51.370877       2.583050        8          386
+FAST f=20 a=7       0.402878       2.583050        8          386
+FAST f=20 a=8       51.437931       2.574875        6          242
+FAST f=20 a=8       0.453094       2.574875        6          242
+FAST f=20 a=9       44.105456       2.576700        6          242
+FAST f=20 a=9       0.456633       2.576700        6          242
+FAST f=20 a=10       44.447580       2.578305        8          338
+FAST f=20 a=10       0.409121       2.578305        8          338
+FAST f=21 a=1       113.031686       2.582449        6          242
+FAST f=21 a=1       1.456971       2.582449        6          242
+FAST f=21 a=2       97.700932       2.582124        8          194
+FAST f=21 a=2       1.072078       2.582124        8          194
+FAST f=21 a=3       96.563648       2.585479        8          434
+FAST f=21 a=3       0.949528       2.585479        8          434
+FAST f=21 a=4       90.597813       2.582366        6          386
+FAST f=21 a=4       0.76944       2.582366        6          386
+FAST f=21 a=5       86.815980       2.579043        8          434
+FAST f=21 a=5       0.858167       2.579043        8          434
+FAST f=21 a=6       91.235820       2.578378        8          530
+FAST f=21 a=6       0.684274       2.578378        8          530
+FAST f=21 a=7       84.392788       2.581243        8          386
+FAST f=21 a=7       0.814386       2.581243        8          386
+FAST f=21 a=8       82.052310       2.582547        8          338
+FAST f=21 a=8       0.822633       2.582547        8          338
+FAST f=21 a=9       74.696074       2.579319        8          194
+FAST f=21 a=9       0.811028       2.579319        8          194
+FAST f=21 a=10       76.211170       2.578766        8          290
+FAST f=21 a=10       0.809715       2.578766        8          290
+FAST f=22 a=1       138.976871       2.580478        8          194
+FAST f=22 a=1       1.748932       2.580478        8          194
+FAST f=22 a=2       120.164097       2.583633        8          386
+FAST f=22 a=2       1.333239       2.583633        8          386
+FAST f=22 a=3       111.986474       2.582566        6          194
+FAST f=22 a=3       1.305734       2.582566        6          194
+FAST f=22 a=4       108.548148       2.583068        6          194
+FAST f=22 a=4       1.314026       2.583068        6          194
+FAST f=22 a=5       103.173017       2.583495        6          290
+FAST f=22 a=5       1.228664       2.583495        6          290
+FAST f=22 a=6       108.421262       2.582349        8          530
+FAST f=22 a=6       1.076773       2.582349        8          530
+FAST f=22 a=7       103.284127       2.581022        8          386
+FAST f=22 a=7       1.112117       2.581022        8          386
+FAST f=22 a=8       96.330279       2.581073        8          290
+FAST f=22 a=8       1.109303       2.581073        8          290
+FAST f=22 a=9       97.651348       2.580075        6          194
+FAST f=22 a=9       0.933032       2.580075        6          194
+FAST f=22 a=10       101.660621       2.584886        8          194
+FAST f=22 a=10       0.796823       2.584886        8          194
+FAST f=23 a=1       159.322978       2.581474        6          242
+FAST f=23 a=1       2.015878       2.581474        6          242
+FAST f=23 a=2       134.331775       2.581619        8          194
+FAST f=23 a=2       1.545845       2.581619        8          194
+FAST f=23 a=3       127.724552       2.579888        6          338
+FAST f=23 a=3       1.444496       2.579888        6          338
+FAST f=23 a=4       126.077675       2.578137        6          242
+FAST f=23 a=4       1.364394       2.578137        6          242
+FAST f=23 a=5       124.914027       2.580843        8          338
+FAST f=23 a=5       1.116059       2.580843        8          338
+FAST f=23 a=6       122.874153       2.577637        6          338
+FAST f=23 a=6       1.164584       2.577637        6          338
+FAST f=23 a=7       123.099257       2.582715        6          386
+FAST f=23 a=7       1.354042       2.582715        6          386
+FAST f=23 a=8       122.026753       2.577681        8          194
+FAST f=23 a=8       1.210966       2.577681        8          194
+FAST f=23 a=9       121.164312       2.584599        6          290
+FAST f=23 a=9       1.174859       2.584599        6          290
+FAST f=23 a=10       117.462222       2.580358        8          194
+FAST f=23 a=10       1.075258       2.580358        8          194
+FAST f=24 a=1       169.539659       2.581642        6          194
+FAST f=24 a=1       1.916804       2.581642        6          194
+FAST f=24 a=2       160.539270       2.580421        6          290
+FAST f=24 a=2       1.71087       2.580421        6          290
+FAST f=24 a=3       155.455874       2.580449        6          242
+FAST f=24 a=3       1.60307       2.580449        6          242
+FAST f=24 a=4       147.630320       2.582953        6          338
+FAST f=24 a=4       1.396364       2.582953        6          338
+FAST f=24 a=5       133.767428       2.580589        6          290
+FAST f=24 a=5       1.19933       2.580589        6          290
+FAST f=24 a=6       146.437535       2.579453        8          194
+FAST f=24 a=6       1.385405       2.579453        8          194
+FAST f=24 a=7       147.227507       2.584155        8          386
+FAST f=24 a=7       1.48942       2.584155        8          386
+FAST f=24 a=8       138.005773       2.584115        8          194
+FAST f=24 a=8       1.352       2.584115        8          194
+FAST f=24 a=9       141.442625       2.582902        8          290
+FAST f=24 a=9       1.39647       2.582902        8          290
+FAST f=24 a=10       142.157446       2.582701        8          434
+FAST f=24 a=10       1.498889       2.582701        8          434
diff --git a/contrib/experimental_dict_builders/benchmarkDictBuilder/benchmark.c b/contrib/experimental_dict_builders/benchmarkDictBuilder/benchmark.c
index d92e8d5cb..b19345692 100644
--- a/contrib/experimental_dict_builders/benchmarkDictBuilder/benchmark.c
+++ b/contrib/experimental_dict_builders/benchmarkDictBuilder/benchmark.c
@@ -5,7 +5,6 @@
 #include 
 #include 
 #include "random.h"
-#include "fastCover.h"
 #include "dictBuilder.h"
 #include "zstd_internal.h" /* includes zstd.h */
 #include "io.h"
@@ -149,7 +148,7 @@ double compressWithDict(sampleInfo *srcInfo, dictInfo* dInfo, int compressionLev
   /* Allocate dst with enough space to compress the maximum sized sample */
   {
     size_t maxSampleSize = 0;
-    for (int i = 0; i < srcInfo->nbSamples; i++) {
+    for (i = 0; i < srcInfo->nbSamples; i++) {
       maxSampleSize = MAX(srcInfo->samplesSizes[i], maxSampleSize);
     }
     dstCapacity = ZSTD_compressBound(maxSampleSize);
@@ -291,6 +290,9 @@ int main(int argCount, const char* argv[])
   /* Initialize arguments to default values */
   unsigned k = 200;
   unsigned d = 8;
+  unsigned f;
+  unsigned accel;
+  unsigned i;
   const unsigned cLevel = DEFAULT_CLEVEL;
   const unsigned dictID = 0;
   const unsigned maxDictSize = g_defaultMaxDictSize;
@@ -305,7 +307,7 @@ int main(int argCount, const char* argv[])
   const char** extendedFileList = NULL;
 
   /* Parse arguments */
-  for (int i = 1; i < argCount; i++) {
+  for (i = 1; i < argCount; i++) {
     const char* argument = argv[i];
     if (longCommandWArg(&argument, "in=")) {
       filenameTable[filenameIdx] = argument;
@@ -375,6 +377,7 @@ int main(int argCount, const char* argv[])
 
   /* for cover */
   {
+    /* for cover (optimizing k and d) */
     ZDICT_cover_params_t coverParam;
     memset(&coverParam, 0, sizeof(coverParam));
     coverParam.zParams = zParams;
@@ -388,6 +391,7 @@ int main(int argCount, const char* argv[])
       goto _cleanup;
     }
 
+    /* for cover (with k and d provided) */
     const int coverResult = benchmarkDictBuilder(srcInfo, maxDictSize, NULL, &coverParam, NULL, NULL);
     DISPLAYLEVEL(2, "k=%u\nd=%u\nsteps=%u\nsplit=%u\n", coverParam.k, coverParam.d, coverParam.steps, (unsigned)(coverParam.splitPoint * 100));
     if(coverResult) {
@@ -398,29 +402,34 @@ int main(int argCount, const char* argv[])
   }
 
   /* for fastCover */
-  for (unsigned f = 15; f < 25; f++){
+  for (f = 15; f < 25; f++){
     DISPLAYLEVEL(2, "current f is %u\n", f);
-    /* for fastCover (optimizing k and d) */
-    ZDICT_fastCover_params_t fastParam;
-    memset(&fastParam, 0, sizeof(fastParam));
-    fastParam.zParams = zParams;
-    fastParam.splitPoint = 1.0;
-    fastParam.f = f;
-    fastParam.steps = 40;
-    fastParam.nbThreads = 1;
-    const int fastOptResult = benchmarkDictBuilder(srcInfo, maxDictSize, NULL, NULL, NULL, &fastParam);
-    DISPLAYLEVEL(2, "k=%u\nd=%u\nf=%u\nsteps=%u\nsplit=%u\n", fastParam.k, fastParam.d, fastParam.f, fastParam.steps, (unsigned)(fastParam.splitPoint * 100));
-    if(fastOptResult) {
-      result = 1;
-      goto _cleanup;
-    }
+    for (accel = 1; accel < 11; accel++) {
+      DISPLAYLEVEL(2, "current accel is %u\n", accel);
+      /* for fastCover (optimizing k and d) */
+      ZDICT_fastCover_params_t fastParam;
+      memset(&fastParam, 0, sizeof(fastParam));
+      fastParam.zParams = zParams;
+      fastParam.f = f;
+      fastParam.steps = 40;
+      fastParam.nbThreads = 1;
+      fastParam.accel = accel;
+      const int fastOptResult = benchmarkDictBuilder(srcInfo, maxDictSize, NULL, NULL, NULL, &fastParam);
+      DISPLAYLEVEL(2, "k=%u\nd=%u\nf=%u\nsteps=%u\nsplit=%u\naccel=%u\n", fastParam.k, fastParam.d, fastParam.f, fastParam.steps, (unsigned)(fastParam.splitPoint * 100), fastParam.accel);
+      if(fastOptResult) {
+        result = 1;
+        goto _cleanup;
+      }
 
-    /* for fastCover (with k and d provided) */
-    const int fastResult = benchmarkDictBuilder(srcInfo, maxDictSize, NULL, NULL, NULL, &fastParam);
-    DISPLAYLEVEL(2, "k=%u\nd=%u\nf=%u\nsteps=%u\nsplit=%u\n", fastParam.k, fastParam.d, fastParam.f, fastParam.steps, (unsigned)(fastParam.splitPoint * 100));
-    if(fastResult) {
-      result = 1;
-      goto _cleanup;
+      /* for fastCover (with k and d provided) */
+      for (i = 0; i < 5; i++) {
+        const int fastResult = benchmarkDictBuilder(srcInfo, maxDictSize, NULL, NULL, NULL, &fastParam);
+        DISPLAYLEVEL(2, "k=%u\nd=%u\nf=%u\nsteps=%u\nsplit=%u\naccel=%u\n", fastParam.k, fastParam.d, fastParam.f, fastParam.steps, (unsigned)(fastParam.splitPoint * 100), fastParam.accel);
+        if(fastResult) {
+          result = 1;
+          goto _cleanup;
+        }
+      }
     }
   }
 
diff --git a/contrib/experimental_dict_builders/fastCover/fastCover.c b/contrib/experimental_dict_builders/fastCover/fastCover.c
index 84d841b10..02c155a81 100644
--- a/contrib/experimental_dict_builders/fastCover/fastCover.c
+++ b/contrib/experimental_dict_builders/fastCover/fastCover.c
@@ -197,7 +197,7 @@ static FASTCOVER_segment_t FASTCOVER_selectSegment(const FASTCOVER_ctx_t *ctx,
     bestSegment.end = newEnd;
   }
   {
-    /* Half the frequency of hash value of each dmer covered by the chosen segment. */
+    /*  Zero the frequency of hash value of each dmer covered by the chosen segment. */
     U32 pos;
     for (pos = bestSegment.begin; pos != bestSegment.end; ++pos) {
       const size_t i = FASTCOVER_hashPtrToIndex(ctx->samples + pos, parameters.f, ctx->d);
@@ -300,7 +300,7 @@ static int FASTCOVER_ctx_init(FASTCOVER_ctx_t *ctx, const void *samplesBuffer,
   if (totalSamplesSize < MAX(d, sizeof(U64)) ||
       totalSamplesSize >= (size_t)FASTCOVER_MAX_SAMPLES_SIZE) {
     DISPLAYLEVEL(1, "Total samples size is too large (%u MB), maximum size is %u MB\n",
-                 (U32)(totalSamplesSize>>20), (FASTCOVER_MAX_SAMPLES_SIZE >> 20));
+                 (U32)(totalSamplesSize >> 20), (FASTCOVER_MAX_SAMPLES_SIZE >> 20));
     return 0;
   }
   /* Check if there are at least 5 training samples */
diff --git a/lib/BUCK b/lib/BUCK
index dbe8885fc..bd93b082a 100644
--- a/lib/BUCK
+++ b/lib/BUCK
@@ -69,6 +69,7 @@ cxx_library(
     ]),
     headers=subdir_glob([
         ('dictBuilder', 'divsufsort.h'),
+        ('dictBuilder', 'cover.h'),
     ]),
     srcs=glob(['dictBuilder/*.c']),
     deps=[':common'],
diff --git a/lib/dictBuilder/cover.c b/lib/dictBuilder/cover.c
index e32991652..74b70ef82 100644
--- a/lib/dictBuilder/cover.c
+++ b/lib/dictBuilder/cover.c
@@ -29,6 +29,7 @@
 #include "mem.h" /* read */
 #include "pool.h"
 #include "threading.h"
+#include "cover.h"
 #include "zstd_internal.h" /* includes zstd.h */
 #ifndef ZDICT_STATIC_LINKING_ONLY
 #define ZDICT_STATIC_LINKING_ONLY
@@ -185,7 +186,7 @@ static void COVER_map_remove(COVER_map_t *map, U32 key) {
 }
 
 /**
- * Destroyes a map that is inited with COVER_map_init().
+ * Destroys a map that is inited with COVER_map_init().
  */
 static void COVER_map_destroy(COVER_map_t *map) {
   if (map->data) {
@@ -223,7 +224,7 @@ static COVER_ctx_t *g_ctx = NULL;
 /**
  * Returns the sum of the sample sizes.
  */
-static size_t COVER_sum(const size_t *samplesSizes, unsigned nbSamples) {
+size_t COVER_sum(const size_t *samplesSizes, unsigned nbSamples) {
   size_t sum = 0;
   unsigned i;
   for (i = 0; i < nbSamples; ++i) {
@@ -380,14 +381,6 @@ static void COVER_group(COVER_ctx_t *ctx, const void *group,
   ctx->suffix[dmerId] = freq;
 }
 
-/**
- * A segment is a range in the source as well as the score of the segment.
- */
-typedef struct {
-  U32 begin;
-  U32 end;
-  U32 score;
-} COVER_segment_t;
 
 /**
  * Selects the best segment in an epoch.
@@ -691,7 +684,7 @@ ZDICTLIB_API size_t ZDICT_trainFromBuffer_cover(
   BYTE* const dict = (BYTE*)dictBuffer;
   COVER_ctx_t ctx;
   COVER_map_t activeDmers;
-  parameters.splitPoint = 1.0;
+  parameters.splitPoint = parameters.splitPoint <= 0.0 ? DEFAULT_SPLITPOINT : parameters.splitPoint;
   /* Initialize global data */
   g_displayLevel = parameters.zParams.notificationLevel;
   /* Checks */
@@ -737,28 +730,65 @@ ZDICTLIB_API size_t ZDICT_trainFromBuffer_cover(
   }
 }
 
-/**
- * COVER_best_t is used for two purposes:
- * 1. Synchronizing threads.
- * 2. Saving the best parameters and dictionary.
- *
- * All of the methods except COVER_best_init() are thread safe if zstd is
- * compiled with multithreaded support.
- */
-typedef struct COVER_best_s {
-  ZSTD_pthread_mutex_t mutex;
-  ZSTD_pthread_cond_t cond;
-  size_t liveJobs;
-  void *dict;
-  size_t dictSize;
-  ZDICT_cover_params_t parameters;
-  size_t compressedSize;
-} COVER_best_t;
+
+
+size_t COVER_checkTotalCompressedSize(const ZDICT_cover_params_t parameters,
+                                    const size_t *samplesSizes, const BYTE *samples,
+                                    size_t *offsets,
+                                    size_t nbTrainSamples, size_t nbSamples,
+                                    BYTE *const dict, size_t dictBufferCapacity) {
+  size_t totalCompressedSize = ERROR(GENERIC);
+  /* Pointers */
+  ZSTD_CCtx *cctx;
+  ZSTD_CDict *cdict;
+  void *dst;
+  /* Local variables */
+  size_t dstCapacity;
+  size_t i;
+  /* Allocate dst with enough space to compress the maximum sized sample */
+  {
+    size_t maxSampleSize = 0;
+    i = parameters.splitPoint < 1.0 ? nbTrainSamples : 0;
+    for (; i < nbSamples; ++i) {
+      maxSampleSize = MAX(samplesSizes[i], maxSampleSize);
+    }
+    dstCapacity = ZSTD_compressBound(maxSampleSize);
+    dst = malloc(dstCapacity);
+  }
+  /* Create the cctx and cdict */
+  cctx = ZSTD_createCCtx();
+  cdict = ZSTD_createCDict(dict, dictBufferCapacity,
+                           parameters.zParams.compressionLevel);
+  if (!dst || !cctx || !cdict) {
+    goto _compressCleanup;
+  }
+  /* Compress each sample and sum their sizes (or error) */
+  totalCompressedSize = dictBufferCapacity;
+  i = parameters.splitPoint < 1.0 ? nbTrainSamples : 0;
+  for (; i < nbSamples; ++i) {
+    const size_t size = ZSTD_compress_usingCDict(
+        cctx, dst, dstCapacity, samples + offsets[i],
+        samplesSizes[i], cdict);
+    if (ZSTD_isError(size)) {
+      totalCompressedSize = ERROR(GENERIC);
+      goto _compressCleanup;
+    }
+    totalCompressedSize += size;
+  }
+_compressCleanup:
+  ZSTD_freeCCtx(cctx);
+  ZSTD_freeCDict(cdict);
+  if (dst) {
+    free(dst);
+  }
+  return totalCompressedSize;
+}
+
 
 /**
  * Initialize the `COVER_best_t`.
  */
-static void COVER_best_init(COVER_best_t *best) {
+void COVER_best_init(COVER_best_t *best) {
   if (best==NULL) return; /* compatible with init on NULL */
   (void)ZSTD_pthread_mutex_init(&best->mutex, NULL);
   (void)ZSTD_pthread_cond_init(&best->cond, NULL);
@@ -772,7 +802,7 @@ static void COVER_best_init(COVER_best_t *best) {
 /**
  * Wait until liveJobs == 0.
  */
-static void COVER_best_wait(COVER_best_t *best) {
+void COVER_best_wait(COVER_best_t *best) {
   if (!best) {
     return;
   }
@@ -786,7 +816,7 @@ static void COVER_best_wait(COVER_best_t *best) {
 /**
  * Call COVER_best_wait() and then destroy the COVER_best_t.
  */
-static void COVER_best_destroy(COVER_best_t *best) {
+void COVER_best_destroy(COVER_best_t *best) {
   if (!best) {
     return;
   }
@@ -802,7 +832,7 @@ static void COVER_best_destroy(COVER_best_t *best) {
  * Called when a thread is about to be launched.
  * Increments liveJobs.
  */
-static void COVER_best_start(COVER_best_t *best) {
+void COVER_best_start(COVER_best_t *best) {
   if (!best) {
     return;
   }
@@ -816,7 +846,7 @@ static void COVER_best_start(COVER_best_t *best) {
  * Decrements liveJobs and signals any waiting threads if liveJobs == 0.
  * If this dictionary is the best so far save it and its parameters.
  */
-static void COVER_best_finish(COVER_best_t *best, size_t compressedSize,
+void COVER_best_finish(COVER_best_t *best, size_t compressedSize,
                               ZDICT_cover_params_t parameters, void *dict,
                               size_t dictSize) {
   if (!best) {
@@ -847,10 +877,10 @@ static void COVER_best_finish(COVER_best_t *best, size_t compressedSize,
       best->parameters = parameters;
       best->compressedSize = compressedSize;
     }
-    ZSTD_pthread_mutex_unlock(&best->mutex);
     if (liveJobs == 0) {
       ZSTD_pthread_cond_broadcast(&best->cond);
     }
+    ZSTD_pthread_mutex_unlock(&best->mutex);
   }
 }
 
@@ -904,51 +934,10 @@ static void COVER_tryParameters(void *opaque) {
     }
   }
   /* Check total compressed size */
-  {
-    /* Pointers */
-    ZSTD_CCtx *cctx;
-    ZSTD_CDict *cdict;
-    void *dst;
-    /* Local variables */
-    size_t dstCapacity;
-    size_t i;
-    /* Allocate dst with enough space to compress the maximum sized sample */
-    {
-      size_t maxSampleSize = 0;
-      i = parameters.splitPoint < 1.0 ? ctx->nbTrainSamples : 0;
-      for (; i < ctx->nbSamples; ++i) {
-        maxSampleSize = MAX(ctx->samplesSizes[i], maxSampleSize);
-      }
-      dstCapacity = ZSTD_compressBound(maxSampleSize);
-      dst = malloc(dstCapacity);
-    }
-    /* Create the cctx and cdict */
-    cctx = ZSTD_createCCtx();
-    cdict = ZSTD_createCDict(dict, dictBufferCapacity,
-                             parameters.zParams.compressionLevel);
-    if (!dst || !cctx || !cdict) {
-      goto _compressCleanup;
-    }
-    /* Compress each sample and sum their sizes (or error) */
-    totalCompressedSize = dictBufferCapacity;
-    i = parameters.splitPoint < 1.0 ? ctx->nbTrainSamples : 0;
-    for (; i < ctx->nbSamples; ++i) {
-      const size_t size = ZSTD_compress_usingCDict(
-          cctx, dst, dstCapacity, ctx->samples + ctx->offsets[i],
-          ctx->samplesSizes[i], cdict);
-      if (ZSTD_isError(size)) {
-        totalCompressedSize = ERROR(GENERIC);
-        goto _compressCleanup;
-      }
-      totalCompressedSize += size;
-    }
-  _compressCleanup:
-    ZSTD_freeCCtx(cctx);
-    ZSTD_freeCDict(cdict);
-    if (dst) {
-      free(dst);
-    }
-  }
+  totalCompressedSize = COVER_checkTotalCompressedSize(parameters, ctx->samplesSizes,
+                                                       ctx->samples, ctx->offsets,
+                                                       ctx->nbTrainSamples, ctx->nbSamples,
+                                                       dict, dictBufferCapacity);
 
 _cleanup:
   COVER_best_finish(data->best, totalCompressedSize, parameters, dict,
diff --git a/lib/dictBuilder/cover.h b/lib/dictBuilder/cover.h
new file mode 100644
index 000000000..82e2e1cea
--- /dev/null
+++ b/lib/dictBuilder/cover.h
@@ -0,0 +1,83 @@
+#include   /* fprintf */
+#include  /* malloc, free, qsort */
+#include  /* memset */
+#include    /* clock */
+#include "mem.h" /* read */
+#include "pool.h"
+#include "threading.h"
+#include "zstd_internal.h" /* includes zstd.h */
+#ifndef ZDICT_STATIC_LINKING_ONLY
+#define ZDICT_STATIC_LINKING_ONLY
+#endif
+#include "zdict.h"
+
+/**
+ * COVER_best_t is used for two purposes:
+ * 1. Synchronizing threads.
+ * 2. Saving the best parameters and dictionary.
+ *
+ * All of the methods except COVER_best_init() are thread safe if zstd is
+ * compiled with multithreaded support.
+ */
+typedef struct COVER_best_s {
+  ZSTD_pthread_mutex_t mutex;
+  ZSTD_pthread_cond_t cond;
+  size_t liveJobs;
+  void *dict;
+  size_t dictSize;
+  ZDICT_cover_params_t parameters;
+  size_t compressedSize;
+} COVER_best_t;
+
+/**
+ * A segment is a range in the source as well as the score of the segment.
+ */
+typedef struct {
+  U32 begin;
+  U32 end;
+  U32 score;
+} COVER_segment_t;
+
+/**
+ *  Checks total compressed size of a dictionary
+ */
+size_t COVER_checkTotalCompressedSize(const ZDICT_cover_params_t parameters,
+                                      const size_t *samplesSizes, const BYTE *samples,
+                                      size_t *offsets,
+                                      size_t nbTrainSamples, size_t nbSamples,
+                                      BYTE *const dict, size_t dictBufferCapacity);
+
+/**
+ * Returns the sum of the sample sizes.
+ */
+size_t COVER_sum(const size_t *samplesSizes, unsigned nbSamples) ;
+
+/**
+ * Initialize the `COVER_best_t`.
+ */
+void COVER_best_init(COVER_best_t *best);
+
+/**
+ * Wait until liveJobs == 0.
+ */
+void COVER_best_wait(COVER_best_t *best);
+
+/**
+ * Call COVER_best_wait() and then destroy the COVER_best_t.
+ */
+void COVER_best_destroy(COVER_best_t *best);
+
+/**
+ * Called when a thread is about to be launched.
+ * Increments liveJobs.
+ */
+void COVER_best_start(COVER_best_t *best);
+
+/**
+ * Called when a thread finishes executing, both on error or success.
+ * Decrements liveJobs and signals any waiting threads if liveJobs == 0.
+ * If this dictionary is the best so far save it and its parameters.
+ */
+void COVER_best_finish(COVER_best_t *best, size_t compressedSize,
+                       ZDICT_cover_params_t parameters, void *dict,
+                       size_t dictSize);
diff --git a/lib/dictBuilder/fastcover.c b/lib/dictBuilder/fastcover.c
new file mode 100644
index 000000000..9a41ac2f9
--- /dev/null
+++ b/lib/dictBuilder/fastcover.c
@@ -0,0 +1,701 @@
+/*-*************************************
+*  Dependencies
+***************************************/
+#include   /* fprintf */
+#include  /* malloc, free, qsort */
+#include  /* memset */
+#include    /* clock */
+
+#include "mem.h" /* read */
+#include "pool.h"
+#include "threading.h"
+#include "cover.h"
+#include "zstd_internal.h" /* includes zstd.h */
+#ifndef ZDICT_STATIC_LINKING_ONLY
+#define ZDICT_STATIC_LINKING_ONLY
+#endif
+#include "zdict.h"
+
+
+/*-*************************************
+*  Constants
+***************************************/
+#define FASTCOVER_MAX_SAMPLES_SIZE (sizeof(size_t) == 8 ? ((U32)-1) : ((U32)1 GB))
+#define FASTCOVER_MAX_F 31
+#define FASTCOVER_MAX_ACCEL 10
+#define DEFAULT_SPLITPOINT 0.75
+#define DEFAULT_F 18
+#define DEFAULT_ACCEL 1
+
+
+/*-*************************************
+*  Console display
+***************************************/
+static int g_displayLevel = 2;
+#define DISPLAY(...)                                                           \
+  {                                                                            \
+    fprintf(stderr, __VA_ARGS__);                                              \
+    fflush(stderr);                                                            \
+  }
+#define LOCALDISPLAYLEVEL(displayLevel, l, ...)                                \
+  if (displayLevel >= l) {                                                     \
+    DISPLAY(__VA_ARGS__);                                                      \
+  } /* 0 : no display;   1: errors;   2: default;  3: details;  4: debug */
+#define DISPLAYLEVEL(l, ...) LOCALDISPLAYLEVEL(g_displayLevel, l, __VA_ARGS__)
+
+#define LOCALDISPLAYUPDATE(displayLevel, l, ...)                               \
+  if (displayLevel >= l) {                                                     \
+    if ((clock() - g_time > refreshRate) || (displayLevel >= 4)) {             \
+      g_time = clock();                                                        \
+      DISPLAY(__VA_ARGS__);                                                    \
+    }                                                                          \
+  }
+#define DISPLAYUPDATE(l, ...) LOCALDISPLAYUPDATE(g_displayLevel, l, __VA_ARGS__)
+static const clock_t refreshRate = CLOCKS_PER_SEC * 15 / 100;
+static clock_t g_time = 0;
+
+
+/*-*************************************
+* Hash Functions
+***************************************/
+static const U64 prime6bytes = 227718039650203ULL;
+static size_t ZSTD_hash6(U64 u, U32 h) { return (size_t)(((u  << (64-48)) * prime6bytes) >> (64-h)) ; }
+static size_t ZSTD_hash6Ptr(const void* p, U32 h) { return ZSTD_hash6(MEM_readLE64(p), h); }
+
+static const U64 prime8bytes = 0xCF1BBCDCB7A56463ULL;
+static size_t ZSTD_hash8(U64 u, U32 h) { return (size_t)(((u) * prime8bytes) >> (64-h)) ; }
+static size_t ZSTD_hash8Ptr(const void* p, U32 h) { return ZSTD_hash8(MEM_readLE64(p), h); }
+
+
+/**
+ * Hash the d-byte value pointed to by p and mod 2^f
+ */
+static size_t FASTCOVER_hashPtrToIndex(const void* p, U32 h, unsigned d) {
+  if (d == 6) {
+    return ZSTD_hash6Ptr(p, h) & ((1 << h) - 1);
+  }
+  return ZSTD_hash8Ptr(p, h) & ((1 << h) - 1);
+}
+
+
+/*-*************************************
+* Acceleration
+***************************************/
+typedef struct {
+  unsigned finalize;    /* Percentage of training samples used for ZDICT_finalizeDictionary */
+  unsigned skip;        /* Number of dmer skipped between each dmer counted in computeFrequency */
+} FASTCOVER_accel_t;
+
+
+static const FASTCOVER_accel_t FASTCOVER_defaultAccelParameters[FASTCOVER_MAX_ACCEL+1] = {
+  { 100, 0 },   /* accel = 0, should not happen because accel = 0 defaults to accel = 1 */
+  { 100, 0 },   /* accel = 1 */
+  { 50, 1 },   /* accel = 2 */
+  { 34, 2 },   /* accel = 3 */
+  { 25, 3 },   /* accel = 4 */
+  { 20, 4 },   /* accel = 5 */
+  { 17, 5 },   /* accel = 6 */
+  { 14, 6 },   /* accel = 7 */
+  { 13, 7 },   /* accel = 8 */
+  { 11, 8 },   /* accel = 9 */
+  { 10, 9 },   /* accel = 10 */
+};
+
+
+/*-*************************************
+* Context
+***************************************/
+typedef struct {
+  const BYTE *samples;
+  size_t *offsets;
+  const size_t *samplesSizes;
+  size_t nbSamples;
+  size_t nbTrainSamples;
+  size_t nbTestSamples;
+  size_t nbDmers;
+  U32 *freqs;
+  unsigned d;
+  unsigned f;
+  FASTCOVER_accel_t accelParams;
+} FASTCOVER_ctx_t;
+
+
+/*-*************************************
+*  Helper functions
+***************************************/
+/**
+ * Selects the best segment in an epoch.
+ * Segments of are scored according to the function:
+ *
+ * Let F(d) be the frequency of all dmers with hash value d.
+ * Let S_i be hash value of the dmer at position i of segment S which has length k.
+ *
+ *     Score(S) = F(S_1) + F(S_2) + ... + F(S_{k-d+1})
+ *
+ * Once the dmer with hash value d is in the dictionay we set F(d) = 0.
+ */
+static COVER_segment_t FASTCOVER_selectSegment(const FASTCOVER_ctx_t *ctx,
+                                              U32 *freqs, U32 begin, U32 end,
+                                              ZDICT_cover_params_t parameters,
+                                              U16* segmentFreqs) {
+  /* Constants */
+  const U32 k = parameters.k;
+  const U32 d = parameters.d;
+  const U32 f = ctx->f;
+  const U32 dmersInK = k - d + 1;
+
+  /* Try each segment (activeSegment) and save the best (bestSegment) */
+  COVER_segment_t bestSegment = {0, 0, 0};
+  COVER_segment_t activeSegment;
+
+  /* Reset the activeDmers in the segment */
+  /* The activeSegment starts at the beginning of the epoch. */
+  activeSegment.begin = begin;
+  activeSegment.end = begin;
+  activeSegment.score = 0;
+
+  /* Slide the activeSegment through the whole epoch.
+   * Save the best segment in bestSegment.
+   */
+  while (activeSegment.end < end) {
+    /* Get hash value of current dmer */
+    const size_t index = FASTCOVER_hashPtrToIndex(ctx->samples + activeSegment.end, f, d);
+
+    /* Add frequency of this index to score if this is the first occurence of index in active segment */
+    if (segmentFreqs[index] == 0) {
+      activeSegment.score += freqs[index];
+    }
+    /* Increment end of segment and segmentFreqs*/
+    activeSegment.end += 1;
+    segmentFreqs[index] += 1;
+    /* If the window is now too large, drop the first position */
+    if (activeSegment.end - activeSegment.begin == dmersInK + 1) {
+      /* Get hash value of the dmer to be eliminated from active segment */
+      const size_t delIndex = FASTCOVER_hashPtrToIndex(ctx->samples + activeSegment.begin, f, d);
+      segmentFreqs[delIndex] -= 1;
+      /* Subtract frequency of this index from score if this is the last occurrence of this index in active segment */
+      if (segmentFreqs[delIndex] == 0) {
+        activeSegment.score -= freqs[delIndex];
+      }
+      /* Increment start of segment */
+      activeSegment.begin += 1;
+    }
+
+    /* If this segment is the best so far save it */
+    if (activeSegment.score > bestSegment.score) {
+      bestSegment = activeSegment;
+    }
+  }
+
+  /* Zero out rest of segmentFreqs array */
+  while (activeSegment.begin < end) {
+    const size_t delIndex = FASTCOVER_hashPtrToIndex(ctx->samples + activeSegment.begin, f, d);
+    segmentFreqs[delIndex] -= 1;
+    activeSegment.begin += 1;
+  }
+
+  {
+    /*  Zero the frequency of hash value of each dmer covered by the chosen segment. */
+    U32 pos;
+    for (pos = bestSegment.begin; pos != bestSegment.end; ++pos) {
+      const size_t i = FASTCOVER_hashPtrToIndex(ctx->samples + pos, f, d);
+      freqs[i] = 0;
+    }
+  }
+
+  return bestSegment;
+}
+
+
+static int FASTCOVER_checkParameters(ZDICT_cover_params_t parameters,
+                                     size_t maxDictSize, unsigned f,
+                                     unsigned accel) {
+  /* k, d, and f are required parameters */
+  if (parameters.d == 0 || parameters.k == 0) {
+    return 0;
+  }
+  /* d has to be 6 or 8 */
+  if (parameters.d != 6 && parameters.d != 8) {
+    return 0;
+  }
+  /* k <= maxDictSize */
+  if (parameters.k > maxDictSize) {
+    return 0;
+  }
+  /* d <= k */
+  if (parameters.d > parameters.k) {
+    return 0;
+  }
+  /* 0 < f <= FASTCOVER_MAX_F*/
+  if (f > FASTCOVER_MAX_F || f == 0) {
+    return 0;
+  }
+  /* 0 < splitPoint <= 1 */
+  if (parameters.splitPoint <= 0 || parameters.splitPoint > 1) {
+    return 0;
+  }
+  /* 0 < accel <= 10 */
+  if (accel > 10 || accel == 0) {
+    return 0;
+  }
+  return 1;
+}
+
+
+/**
+ * Clean up a context initialized with `FASTCOVER_ctx_init()`.
+ */
+static void FASTCOVER_ctx_destroy(FASTCOVER_ctx_t *ctx) {
+  if (!ctx) {
+    return;
+  }
+
+  free(ctx->freqs);
+  ctx->freqs = NULL;
+
+  free(ctx->offsets);
+  ctx->offsets = NULL;
+}
+
+
+/**
+ * Calculate for frequency of hash value of each dmer in ctx->samples
+ */
+static void FASTCOVER_computeFrequency(U32 *freqs, FASTCOVER_ctx_t *ctx){
+  const unsigned f = ctx->f;
+  const unsigned d = ctx->d;
+  const unsigned skip = ctx->accelParams.skip;
+  const unsigned readLength = MAX(d, 8);
+  size_t start; /* start of current dmer */
+  size_t i;
+  for (i = 0; i < ctx->nbTrainSamples; i++) {
+    size_t currSampleStart = ctx->offsets[i];
+    size_t currSampleEnd = ctx->offsets[i+1];
+    start = currSampleStart;
+    while (start + readLength <= currSampleEnd) {
+      const size_t dmerIndex = FASTCOVER_hashPtrToIndex(ctx->samples + start, f, d);
+      freqs[dmerIndex]++;
+      start = start + skip + 1;
+    }
+  }
+}
+
+
+/**
+ * Prepare a context for dictionary building.
+ * The context is only dependent on the parameter `d` and can used multiple
+ * times.
+ * Returns 1 on success or zero on error.
+ * The context must be destroyed with `FASTCOVER_ctx_destroy()`.
+ */
+static int FASTCOVER_ctx_init(FASTCOVER_ctx_t *ctx, const void *samplesBuffer,
+                              const size_t *samplesSizes, unsigned nbSamples,
+                              unsigned d, double splitPoint, unsigned f,
+                              FASTCOVER_accel_t accelParams) {
+  const BYTE *const samples = (const BYTE *)samplesBuffer;
+  const size_t totalSamplesSize = COVER_sum(samplesSizes, nbSamples);
+  /* Split samples into testing and training sets */
+  const unsigned nbTrainSamples = splitPoint < 1.0 ? (unsigned)((double)nbSamples * splitPoint) : nbSamples;
+  const unsigned nbTestSamples = splitPoint < 1.0 ? nbSamples - nbTrainSamples : nbSamples;
+  const size_t trainingSamplesSize = splitPoint < 1.0 ? COVER_sum(samplesSizes, nbTrainSamples) : totalSamplesSize;
+  const size_t testSamplesSize = splitPoint < 1.0 ? COVER_sum(samplesSizes + nbTrainSamples, nbTestSamples) : totalSamplesSize;
+  /* Checks */
+  if (totalSamplesSize < MAX(d, sizeof(U64)) ||
+      totalSamplesSize >= (size_t)FASTCOVER_MAX_SAMPLES_SIZE) {
+    DISPLAYLEVEL(1, "Total samples size is too large (%u MB), maximum size is %u MB\n",
+                 (U32)(totalSamplesSize >> 20), (FASTCOVER_MAX_SAMPLES_SIZE >> 20));
+    return 0;
+  }
+  /* Check if there are at least 5 training samples */
+  if (nbTrainSamples < 5) {
+    DISPLAYLEVEL(1, "Total number of training samples is %u and is invalid\n", nbTrainSamples);
+    return 0;
+  }
+  /* Check if there's testing sample */
+  if (nbTestSamples < 1) {
+    DISPLAYLEVEL(1, "Total number of testing samples is %u and is invalid.\n", nbTestSamples);
+    return 0;
+  }
+  /* Zero the context */
+  memset(ctx, 0, sizeof(*ctx));
+  DISPLAYLEVEL(2, "Training on %u samples of total size %u\n", nbTrainSamples,
+               (U32)trainingSamplesSize);
+  DISPLAYLEVEL(2, "Testing on %u samples of total size %u\n", nbTestSamples,
+               (U32)testSamplesSize);
+
+  ctx->samples = samples;
+  ctx->samplesSizes = samplesSizes;
+  ctx->nbSamples = nbSamples;
+  ctx->nbTrainSamples = nbTrainSamples;
+  ctx->nbTestSamples = nbTestSamples;
+  ctx->nbDmers = trainingSamplesSize - MAX(d, sizeof(U64)) + 1;
+  ctx->d = d;
+  ctx->f = f;
+  ctx->accelParams = accelParams;
+
+  /* The offsets of each file */
+  ctx->offsets = (size_t *)malloc((nbSamples + 1) * sizeof(size_t));
+  if (!ctx->offsets) {
+    DISPLAYLEVEL(1, "Failed to allocate scratch buffers\n");
+    FASTCOVER_ctx_destroy(ctx);
+    return 0;
+  }
+
+  /* Fill offsets from the samplesSizes */
+  {
+    U32 i;
+    ctx->offsets[0] = 0;
+    for (i = 1; i <= nbSamples; ++i) {
+      ctx->offsets[i] = ctx->offsets[i - 1] + samplesSizes[i - 1];
+    }
+  }
+
+  /* Initialize frequency array of size 2^f */
+  ctx->freqs = (U32 *)calloc(((U64)1 << f), sizeof(U32));
+
+  DISPLAYLEVEL(2, "Computing frequencies\n");
+  FASTCOVER_computeFrequency(ctx->freqs, ctx);
+
+  return 1;
+}
+
+
+/**
+ * Given the prepared context build the dictionary.
+ */
+static size_t FASTCOVER_buildDictionary(const FASTCOVER_ctx_t *ctx, U32 *freqs,
+                                        void *dictBuffer, size_t dictBufferCapacity,
+                                        ZDICT_cover_params_t parameters, U16* segmentFreqs){
+  BYTE *const dict = (BYTE *)dictBuffer;
+  size_t tail = dictBufferCapacity;
+  /* Divide the data up into epochs of equal size.
+   * We will select at least one segment from each epoch.
+   */
+  const U32 epochs = MAX(1, (U32)(dictBufferCapacity / parameters.k));
+  const U32 epochSize = (U32)(ctx->nbDmers / epochs);
+  size_t epoch;
+  DISPLAYLEVEL(2, "Breaking content into %u epochs of size %u\n", epochs,
+               epochSize);
+  /* Loop through the epochs until there are no more segments or the dictionary
+   * is full.
+   */
+  for (epoch = 0; tail > 0; epoch = (epoch + 1) % epochs) {
+    const U32 epochBegin = (U32)(epoch * epochSize);
+    const U32 epochEnd = epochBegin + epochSize;
+    size_t segmentSize;
+    /* Select a segment */
+    COVER_segment_t segment = FASTCOVER_selectSegment(
+        ctx, freqs, epochBegin, epochEnd, parameters, segmentFreqs);
+
+    /* If the segment covers no dmers, then we are out of content */
+    if (segment.score == 0) {
+      break;
+    }
+
+    /* Trim the segment if necessary and if it is too small then we are done */
+    segmentSize = MIN(segment.end - segment.begin + parameters.d - 1, tail);
+    if (segmentSize < parameters.d) {
+      break;
+    }
+
+    /* We fill the dictionary from the back to allow the best segments to be
+     * referenced with the smallest offsets.
+     */
+    tail -= segmentSize;
+    memcpy(dict + tail, ctx->samples + segment.begin, segmentSize);
+    DISPLAYUPDATE(
+        2, "\r%u%%       ",
+        (U32)(((dictBufferCapacity - tail) * 100) / dictBufferCapacity));
+  }
+  DISPLAYLEVEL(2, "\r%79s\r", "");
+  return tail;
+}
+
+
+/**
+ * Parameters for FASTCOVER_tryParameters().
+ */
+typedef struct FASTCOVER_tryParameters_data_s {
+  const FASTCOVER_ctx_t *ctx;
+  COVER_best_t *best;
+  size_t dictBufferCapacity;
+  ZDICT_cover_params_t parameters;
+} FASTCOVER_tryParameters_data_t;
+
+
+/**
+ * Tries a set of parameters and updates the COVER_best_t with the results.
+ * This function is thread safe if zstd is compiled with multithreaded support.
+ * It takes its parameters as an *OWNING* opaque pointer to support threading.
+ */
+static void FASTCOVER_tryParameters(void *opaque) {
+  /* Save parameters as local variables */
+  FASTCOVER_tryParameters_data_t *const data = (FASTCOVER_tryParameters_data_t *)opaque;
+  const FASTCOVER_ctx_t *const ctx = data->ctx;
+  const ZDICT_cover_params_t parameters = data->parameters;
+  size_t dictBufferCapacity = data->dictBufferCapacity;
+  size_t totalCompressedSize = ERROR(GENERIC);
+  /* Initialize array to keep track of frequency of dmer within activeSegment */
+  U16* segmentFreqs = (U16 *)calloc(((U64)1 << ctx->f), sizeof(U16));
+  /* Allocate space for hash table, dict, and freqs */
+  BYTE *const dict = (BYTE * const)malloc(dictBufferCapacity);
+  U32 *freqs = (U32*) malloc(((U64)1 << ctx->f) * sizeof(U32));
+  if (!segmentFreqs || !dict || !freqs) {
+    DISPLAYLEVEL(1, "Failed to allocate buffers: out of memory\n");
+    goto _cleanup;
+  }
+  /* Copy the frequencies because we need to modify them */
+  memcpy(freqs, ctx->freqs, ((U64)1 << ctx->f) * sizeof(U32));
+  /* Build the dictionary */
+  {
+    const size_t tail = FASTCOVER_buildDictionary(ctx, freqs, dict, dictBufferCapacity,
+                                                  parameters, segmentFreqs);
+    const unsigned nbFinalizeSamples = (unsigned)(ctx->nbTrainSamples * ctx->accelParams.finalize / 100);
+    dictBufferCapacity = ZDICT_finalizeDictionary(
+        dict, dictBufferCapacity, dict + tail, dictBufferCapacity - tail,
+        ctx->samples, ctx->samplesSizes, nbFinalizeSamples, parameters.zParams);
+    if (ZDICT_isError(dictBufferCapacity)) {
+      DISPLAYLEVEL(1, "Failed to finalize dictionary\n");
+      goto _cleanup;
+    }
+  }
+  /* Check total compressed size */
+  totalCompressedSize = COVER_checkTotalCompressedSize(parameters, ctx->samplesSizes,
+                                                       ctx->samples, ctx->offsets,
+                                                       ctx->nbTrainSamples, ctx->nbSamples,
+                                                       dict, dictBufferCapacity);
+_cleanup:
+  COVER_best_finish(data->best, totalCompressedSize, parameters, dict,
+                    dictBufferCapacity);
+  free(data);
+  free(segmentFreqs);
+  free(dict);
+  free(freqs);
+}
+
+
+
+static void FASTCOVER_convertToCoverParams(ZDICT_fastCover_params_t fastCoverParams,
+                                          ZDICT_cover_params_t *coverParams) {
+    coverParams->k = fastCoverParams.k;
+    coverParams->d = fastCoverParams.d;
+    coverParams->steps = fastCoverParams.steps;
+    coverParams->nbThreads = fastCoverParams.nbThreads;
+    coverParams->splitPoint = fastCoverParams.splitPoint;
+    coverParams->zParams = fastCoverParams.zParams;
+}
+
+
+static void FASTCOVER_convertToFastCoverParams(ZDICT_cover_params_t coverParams,
+                                          ZDICT_fastCover_params_t *fastCoverParams,
+                                          unsigned f, unsigned accel) {
+    fastCoverParams->k = coverParams.k;
+    fastCoverParams->d = coverParams.d;
+    fastCoverParams->steps = coverParams.steps;
+    fastCoverParams->nbThreads = coverParams.nbThreads;
+    fastCoverParams->splitPoint = coverParams.splitPoint;
+    fastCoverParams->f = f;
+    fastCoverParams->accel = accel;
+    fastCoverParams->zParams = coverParams.zParams;
+}
+
+
+ZDICTLIB_API size_t ZDICT_trainFromBuffer_fastCover(
+    void *dictBuffer, size_t dictBufferCapacity, const void *samplesBuffer,
+    const size_t *samplesSizes, unsigned nbSamples, ZDICT_fastCover_params_t parameters) {
+    BYTE* const dict = (BYTE*)dictBuffer;
+    FASTCOVER_ctx_t ctx;
+    ZDICT_cover_params_t coverParams;
+    FASTCOVER_accel_t accelParams;
+    /* Initialize global data */
+    g_displayLevel = parameters.zParams.notificationLevel;
+    /* Assign splitPoint and f if not provided */
+    parameters.splitPoint = parameters.splitPoint <= 0.0 ? DEFAULT_SPLITPOINT : parameters.splitPoint;
+    parameters.f = parameters.f == 0 ? DEFAULT_F : parameters.f;
+    parameters.accel = parameters.accel == 0 ? DEFAULT_ACCEL : parameters.accel;
+    /* Convert to cover parameter */
+    memset(&coverParams, 0 , sizeof(coverParams));
+    FASTCOVER_convertToCoverParams(parameters, &coverParams);
+    /* Checks */
+    if (!FASTCOVER_checkParameters(coverParams, dictBufferCapacity, parameters.f,
+                                   parameters.accel)) {
+      DISPLAYLEVEL(1, "FASTCOVER parameters incorrect\n");
+      return ERROR(GENERIC);
+    }
+    if (nbSamples == 0) {
+      DISPLAYLEVEL(1, "FASTCOVER must have at least one input file\n");
+      return ERROR(GENERIC);
+    }
+    if (dictBufferCapacity < ZDICT_DICTSIZE_MIN) {
+      DISPLAYLEVEL(1, "dictBufferCapacity must be at least %u\n",
+                   ZDICT_DICTSIZE_MIN);
+      return ERROR(dstSize_tooSmall);
+    }
+    /* Assign corresponding FASTCOVER_accel_t to accelParams*/
+    accelParams = FASTCOVER_defaultAccelParameters[parameters.accel];
+    /* Initialize context */
+    if (!FASTCOVER_ctx_init(&ctx, samplesBuffer, samplesSizes, nbSamples,
+                            coverParams.d, parameters.splitPoint, parameters.f,
+                            accelParams)) {
+      DISPLAYLEVEL(1, "Failed to initialize context\n");
+      return ERROR(GENERIC);
+    }
+    /* Build the dictionary */
+    DISPLAYLEVEL(2, "Building dictionary\n");
+    {
+      /* Initialize array to keep track of frequency of dmer within activeSegment */
+      U16* segmentFreqs = (U16 *)calloc(((U64)1 << parameters.f), sizeof(U16));
+      const size_t tail = FASTCOVER_buildDictionary(&ctx, ctx.freqs, dictBuffer,
+                                                dictBufferCapacity, coverParams, segmentFreqs);
+      const unsigned nbFinalizeSamples = (unsigned)(ctx.nbTrainSamples * ctx.accelParams.finalize / 100);
+      const size_t dictionarySize = ZDICT_finalizeDictionary(
+          dict, dictBufferCapacity, dict + tail, dictBufferCapacity - tail,
+          samplesBuffer, samplesSizes, nbFinalizeSamples, coverParams.zParams);
+      if (!ZSTD_isError(dictionarySize)) {
+          DISPLAYLEVEL(2, "Constructed dictionary of size %u\n",
+                      (U32)dictionarySize);
+      }
+      FASTCOVER_ctx_destroy(&ctx);
+      free(segmentFreqs);
+      return dictionarySize;
+    }
+}
+
+
+ZDICTLIB_API size_t ZDICT_optimizeTrainFromBuffer_fastCover(
+    void *dictBuffer, size_t dictBufferCapacity, const void *samplesBuffer,
+    const size_t *samplesSizes, unsigned nbSamples,
+    ZDICT_fastCover_params_t *parameters) {
+    ZDICT_cover_params_t coverParams;
+    FASTCOVER_accel_t accelParams;
+    /* constants */
+    const unsigned nbThreads = parameters->nbThreads;
+    const double splitPoint =
+        parameters->splitPoint <= 0.0 ? DEFAULT_SPLITPOINT : parameters->splitPoint;
+    const unsigned kMinD = parameters->d == 0 ? 6 : parameters->d;
+    const unsigned kMaxD = parameters->d == 0 ? 8 : parameters->d;
+    const unsigned kMinK = parameters->k == 0 ? 50 : parameters->k;
+    const unsigned kMaxK = parameters->k == 0 ? 2000 : parameters->k;
+    const unsigned kSteps = parameters->steps == 0 ? 40 : parameters->steps;
+    const unsigned kStepSize = MAX((kMaxK - kMinK) / kSteps, 1);
+    const unsigned kIterations =
+        (1 + (kMaxD - kMinD) / 2) * (1 + (kMaxK - kMinK) / kStepSize);
+    const unsigned f = parameters->f == 0 ? DEFAULT_F : parameters->f;
+    const unsigned accel = parameters->accel == 0 ? DEFAULT_ACCEL : parameters->accel;
+    /* Local variables */
+    const int displayLevel = parameters->zParams.notificationLevel;
+    unsigned iteration = 1;
+    unsigned d;
+    unsigned k;
+    COVER_best_t best;
+    POOL_ctx *pool = NULL;
+    /* Checks */
+    if (splitPoint <= 0 || splitPoint > 1) {
+      LOCALDISPLAYLEVEL(displayLevel, 1, "Incorrect splitPoint\n");
+      return ERROR(GENERIC);
+    }
+    if (accel == 0 || accel > FASTCOVER_MAX_ACCEL) {
+      LOCALDISPLAYLEVEL(displayLevel, 1, "Incorrect accel\n");
+      return ERROR(GENERIC);
+    }
+    if (kMinK < kMaxD || kMaxK < kMinK) {
+      LOCALDISPLAYLEVEL(displayLevel, 1, "Incorrect k\n");
+      return ERROR(GENERIC);
+    }
+    if (nbSamples == 0) {
+      LOCALDISPLAYLEVEL(displayLevel, 1, "FASTCOVER must have at least one input file\n");
+      return ERROR(GENERIC);
+    }
+    if (dictBufferCapacity < ZDICT_DICTSIZE_MIN) {
+      LOCALDISPLAYLEVEL(displayLevel, 1, "dictBufferCapacity must be at least %u\n",
+                   ZDICT_DICTSIZE_MIN);
+      return ERROR(dstSize_tooSmall);
+    }
+    if (nbThreads > 1) {
+      pool = POOL_create(nbThreads, 1);
+      if (!pool) {
+        return ERROR(memory_allocation);
+      }
+    }
+    /* Initialization */
+    COVER_best_init(&best);
+    memset(&coverParams, 0 , sizeof(coverParams));
+    FASTCOVER_convertToCoverParams(*parameters, &coverParams);
+    accelParams = FASTCOVER_defaultAccelParameters[accel];
+    /* Turn down global display level to clean up display at level 2 and below */
+    g_displayLevel = displayLevel == 0 ? 0 : displayLevel - 1;
+    /* Loop through d first because each new value needs a new context */
+    LOCALDISPLAYLEVEL(displayLevel, 2, "Trying %u different sets of parameters\n",
+                      kIterations);
+    for (d = kMinD; d <= kMaxD; d += 2) {
+      /* Initialize the context for this value of d */
+      FASTCOVER_ctx_t ctx;
+      LOCALDISPLAYLEVEL(displayLevel, 3, "d=%u\n", d);
+      if (!FASTCOVER_ctx_init(&ctx, samplesBuffer, samplesSizes, nbSamples, d, splitPoint, f, accelParams)) {
+        LOCALDISPLAYLEVEL(displayLevel, 1, "Failed to initialize context\n");
+        COVER_best_destroy(&best);
+        POOL_free(pool);
+        return ERROR(GENERIC);
+      }
+      /* Loop through k reusing the same context */
+      for (k = kMinK; k <= kMaxK; k += kStepSize) {
+        /* Prepare the arguments */
+        FASTCOVER_tryParameters_data_t *data = (FASTCOVER_tryParameters_data_t *)malloc(
+            sizeof(FASTCOVER_tryParameters_data_t));
+        LOCALDISPLAYLEVEL(displayLevel, 3, "k=%u\n", k);
+        if (!data) {
+          LOCALDISPLAYLEVEL(displayLevel, 1, "Failed to allocate parameters\n");
+          COVER_best_destroy(&best);
+          FASTCOVER_ctx_destroy(&ctx);
+          POOL_free(pool);
+          return ERROR(GENERIC);
+        }
+        data->ctx = &ctx;
+        data->best = &best;
+        data->dictBufferCapacity = dictBufferCapacity;
+        data->parameters = coverParams;
+        data->parameters.k = k;
+        data->parameters.d = d;
+        data->parameters.splitPoint = splitPoint;
+        data->parameters.steps = kSteps;
+        data->parameters.zParams.notificationLevel = g_displayLevel;
+        /* Check the parameters */
+        if (!FASTCOVER_checkParameters(data->parameters, dictBufferCapacity,
+                                       data->ctx->f, accel)) {
+          DISPLAYLEVEL(1, "FASTCOVER parameters incorrect\n");
+          free(data);
+          continue;
+        }
+        /* Call the function and pass ownership of data to it */
+        COVER_best_start(&best);
+        if (pool) {
+          POOL_add(pool, &FASTCOVER_tryParameters, data);
+        } else {
+          FASTCOVER_tryParameters(data);
+        }
+        /* Print status */
+        LOCALDISPLAYUPDATE(displayLevel, 2, "\r%u%%       ",
+                           (U32)((iteration * 100) / kIterations));
+        ++iteration;
+      }
+      COVER_best_wait(&best);
+      FASTCOVER_ctx_destroy(&ctx);
+    }
+    LOCALDISPLAYLEVEL(displayLevel, 2, "\r%79s\r", "");
+    /* Fill the output buffer and parameters with output of the best parameters */
+    {
+      const size_t dictSize = best.dictSize;
+      if (ZSTD_isError(best.compressedSize)) {
+        const size_t compressedSize = best.compressedSize;
+        COVER_best_destroy(&best);
+        POOL_free(pool);
+        return compressedSize;
+      }
+      FASTCOVER_convertToFastCoverParams(best.parameters, parameters, f, accel);
+      memcpy(dictBuffer, best.dict, dictSize);
+      COVER_best_destroy(&best);
+      POOL_free(pool);
+      return dictSize;
+    }
+
+}
diff --git a/lib/dictBuilder/zdict.c b/lib/dictBuilder/zdict.c
index a4d0a4481..9acceb40b 100644
--- a/lib/dictBuilder/zdict.c
+++ b/lib/dictBuilder/zdict.c
@@ -863,8 +863,8 @@ _cleanup:
 
 size_t ZDICT_finalizeDictionary(void* dictBuffer, size_t dictBufferCapacity,
                           const void* customDictContent, size_t dictContentSize,
-                          const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples,
-                          ZDICT_params_t params)
+                          const void* samplesBuffer, const size_t* samplesSizes,
+                          unsigned nbSamples, ZDICT_params_t params)
 {
     size_t hSize;
 #define HBUFFSIZE 256   /* should prove large enough for all entropy headers */
@@ -1078,17 +1078,17 @@ size_t ZDICT_trainFromBuffer_legacy(void* dictBuffer, size_t dictBufferCapacity,
 size_t ZDICT_trainFromBuffer(void* dictBuffer, size_t dictBufferCapacity,
                              const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples)
 {
-    ZDICT_cover_params_t params;
+    ZDICT_fastCover_params_t params;
     DEBUGLOG(3, "ZDICT_trainFromBuffer");
     memset(¶ms, 0, sizeof(params));
     params.d = 8;
     params.steps = 4;
     /* Default to level 6 since no compression level information is available */
-    params.zParams.compressionLevel = 6;
+    params.zParams.compressionLevel = 3;
 #if defined(DEBUGLEVEL) && (DEBUGLEVEL>=1)
     params.zParams.notificationLevel = DEBUGLEVEL;
 #endif
-    return ZDICT_optimizeTrainFromBuffer_cover(dictBuffer, dictBufferCapacity,
+    return ZDICT_optimizeTrainFromBuffer_fastCover(dictBuffer, dictBufferCapacity,
                                                samplesBuffer, samplesSizes, nbSamples,
                                                ¶ms);
 }
diff --git a/lib/dictBuilder/zdict.h b/lib/dictBuilder/zdict.h
index 4094669d1..c838fd4a7 100644
--- a/lib/dictBuilder/zdict.h
+++ b/lib/dictBuilder/zdict.h
@@ -39,7 +39,8 @@ extern "C" {
 
 /*! ZDICT_trainFromBuffer():
  *  Train a dictionary from an array of samples.
- *  Redirect towards ZDICT_optimizeTrainFromBuffer_cover() single-threaded, with d=8 and steps=4.
+ *  Redirect towards ZDICT_optimizeTrainFromBuffer_fastCover() single-threaded, with d=8, steps=4,
+ *  f=18, and accel=1.
  *  Samples must be stored concatenated in a single flat buffer `samplesBuffer`,
  *  supplied with an array of sizes `samplesSizes`, providing the size of each sample, in order.
  *  The resulting dictionary will be saved into `dictBuffer`.
@@ -90,6 +91,16 @@ typedef struct {
     ZDICT_params_t zParams;
 } ZDICT_cover_params_t;
 
+typedef struct {
+    unsigned k;                  /* Segment size : constraint: 0 < k : Reasonable range [16, 2048+] */
+    unsigned d;                  /* dmer size : constraint: 0 < d <= k : Reasonable range [6, 16] */
+    unsigned f;                  /* log of size of frequency array : constraint: 0 < f <= 31 : 1 means default(18)*/
+    unsigned steps;              /* Number of steps : Only used for optimization : 0 means default (32) : Higher means more parameters checked */
+    unsigned nbThreads;          /* Number of threads : constraint: 0 < nbThreads : 1 means single-threaded : Only used for optimization : Ignored if ZSTD_MULTITHREAD is not defined */
+    double splitPoint;           /* Percentage of samples used for training: the first nbSamples * splitPoint samples will be used to training, the last nbSamples * (1 - splitPoint) samples will be used for testing, 0 means default (0.75), 1.0 when all samples are used for both training and testing */
+    unsigned accel;              /* Acceleration level: constraint: 0 < accel <= 10, higher means faster and less accurate, 0 means default(1) */
+    ZDICT_params_t zParams;
+} ZDICT_fastCover_params_t;
 
 /*! ZDICT_trainFromBuffer_cover():
  *  Train a dictionary from an array of samples using the COVER algorithm.
@@ -116,9 +127,9 @@ ZDICTLIB_API size_t ZDICT_trainFromBuffer_cover(
  * dictionary constructed with those parameters is stored in `dictBuffer`.
  *
  * All of the parameters d, k, steps are optional.
- * If d is non-zero then we don't check multiple values of d, otherwise we check d = {6, 8, 10, 12, 14, 16}.
+ * If d is non-zero then we don't check multiple values of d, otherwise we check d = {6, 8}.
  * if steps is zero it defaults to its default value.
- * If k is non-zero then we don't check multiple values of k, otherwise we check steps values in [16, 2048].
+ * If k is non-zero then we don't check multiple values of k, otherwise we check steps values in [50, 2000].
  *
  * @return: size of dictionary stored into `dictBuffer` (<= `dictBufferCapacity`)
  *           or an error code, which can be tested with ZDICT_isError().
@@ -130,6 +141,48 @@ ZDICTLIB_API size_t ZDICT_optimizeTrainFromBuffer_cover(
     const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples,
           ZDICT_cover_params_t* parameters);
 
+/*! ZDICT_trainFromBuffer_fastCover():
+ *  Train a dictionary from an array of samples using a modified version of COVER algorithm.
+ *  Samples must be stored concatenated in a single flat buffer `samplesBuffer`,
+ *  supplied with an array of sizes `samplesSizes`, providing the size of each sample, in order.
+ *  d and k are required.
+ *  All other parameters are optional, will use default values if not provided
+ *  The resulting dictionary will be saved into `dictBuffer`.
+ * @return: size of dictionary stored into `dictBuffer` (<= `dictBufferCapacity`)
+ *          or an error code, which can be tested with ZDICT_isError().
+ *  Note: ZDICT_trainFromBuffer_fastCover() requires about 1 bytes of memory for each input byte and additionally another 6 * 2^f bytes of memory .
+ *  Tips: In general, a reasonable dictionary has a size of ~ 100 KB.
+ *        It's possible to select smaller or larger size, just by specifying `dictBufferCapacity`.
+ *        In general, it's recommended to provide a few thousands samples, though this can vary a lot.
+ *        It's recommended that total size of all samples be about ~x100 times the target size of dictionary.
+ */
+ZDICTLIB_API size_t ZDICT_trainFromBuffer_fastCover(void *dictBuffer,
+                    size_t dictBufferCapacity, const void *samplesBuffer,
+                    const size_t *samplesSizes, unsigned nbSamples,
+                    ZDICT_fastCover_params_t parameters);
+
+/*! ZDICT_optimizeTrainFromBuffer_fastCover():
+ * The same requirements as above hold for all the parameters except `parameters`.
+ * This function tries many parameter combinations (specifically, k and d combinations)
+ * and picks the best parameters. `*parameters` is filled with the best parameters found,
+ * dictionary constructed with those parameters is stored in `dictBuffer`.
+ * All of the parameters d, k, steps, f, and accel are optional.
+ * If d is non-zero then we don't check multiple values of d, otherwise we check d = {6, 8}.
+ * if steps is zero it defaults to its default value.
+ * If k is non-zero then we don't check multiple values of k, otherwise we check steps values in [50, 2000].
+ * If f is zero, default value of 18 is used.
+ * If accel is zero, default value of 1 is used.
+ *
+ * @return: size of dictionary stored into `dictBuffer` (<= `dictBufferCapacity`)
+ *           or an error code, which can be tested with ZDICT_isError().
+ *           On success `*parameters` contains the parameters selected.
+ * Note: ZDICT_optimizeTrainFromBuffer_fastCover() requires about 1 byte of memory for each input byte and additionally another 6 * 2^f bytes of memory for each thread.
+ */
+ZDICTLIB_API size_t ZDICT_optimizeTrainFromBuffer_fastCover(void* dictBuffer,
+                    size_t dictBufferCapacity, const void* samplesBuffer,
+                    const size_t* samplesSizes, unsigned nbSamples,
+                    ZDICT_fastCover_params_t* parameters);
+
 /*! ZDICT_finalizeDictionary():
  * Given a custom content as a basis for dictionary, and a set of samples,
  * finalize dictionary by adding headers and statistics.
diff --git a/programs/README.md b/programs/README.md
index 22a00409c..0fa033ccf 100644
--- a/programs/README.md
+++ b/programs/README.md
@@ -151,6 +151,7 @@ Advanced arguments :
 Dictionary builder :
 --train ## : create a dictionary from a training set of files
 --train-cover[=k=#,d=#,steps=#,split=#] : use the cover algorithm with optional args
+--train-fastcover[=k=#,d=#,f=#,steps=#,split=#,accel=#] : use the fastcover algorithm with optional args
 --train-legacy[=s=#] : use the legacy algorithm with selectivity (default: 9)
  -o file : `file` is dictionary name (default: dictionary)
 --maxdict=# : limit dictionary to specified size (default: 112640)
diff --git a/programs/dibio.c b/programs/dibio.c
index fbb8aa6fa..4b68be6c9 100644
--- a/programs/dibio.c
+++ b/programs/dibio.c
@@ -44,6 +44,7 @@
 #define SAMPLESIZE_MAX (128 KB)
 #define MEMMULT 11    /* rough estimation : memory cost to analyze 1 byte of sample */
 #define COVER_MEMMULT 9    /* rough estimation : memory cost to analyze 1 byte of sample */
+#define FASTCOVER_MEMMULT 1    /* rough estimation : memory cost to analyze 1 byte of sample */
 static const size_t g_maxMemory = (sizeof(size_t) == 4) ? (2 GB - 64 MB) : ((size_t)(512 MB) << sizeof(size_t));
 
 #define NOISELENGTH 32
@@ -271,16 +272,19 @@ size_t ZDICT_trainFromBuffer_unsafe_legacy(void* dictBuffer, size_t dictBufferCa
 
 int DiB_trainFromFiles(const char* dictFileName, unsigned maxDictSize,
                        const char** fileNamesTable, unsigned nbFiles, size_t chunkSize,
-                       ZDICT_legacy_params_t *params, ZDICT_cover_params_t *coverParams,
-                       int optimizeCover)
+                       ZDICT_legacy_params_t* params, ZDICT_cover_params_t* coverParams,
+                       ZDICT_fastCover_params_t* fastCoverParams, int optimize)
 {
     unsigned const displayLevel = params ? params->zParams.notificationLevel :
                         coverParams ? coverParams->zParams.notificationLevel :
+                        fastCoverParams ? fastCoverParams->zParams.notificationLevel :
                         0;   /* should never happen */
     void* const dictBuffer = malloc(maxDictSize);
     fileStats const fs = DiB_fileStats(fileNamesTable, nbFiles, chunkSize, displayLevel);
     size_t* const sampleSizes = (size_t*)malloc(fs.nbSamples * sizeof(size_t));
-    size_t const memMult = params ? MEMMULT : COVER_MEMMULT;
+    size_t const memMult = params ? MEMMULT :
+                           coverParams ? COVER_MEMMULT:
+                           FASTCOVER_MEMMULT;
     size_t const maxMem =  DiB_findMaxMem(fs.totalSizeToLoad * memMult) / memMult;
     size_t loadedSize = (size_t) MIN ((unsigned long long)maxMem, fs.totalSizeToLoad);
     void* const srcBuffer = malloc(loadedSize+NOISELENGTH);
@@ -312,6 +316,7 @@ int DiB_trainFromFiles(const char* dictFileName, unsigned maxDictSize,
     /* Load input buffer */
     DISPLAYLEVEL(3, "Shuffling input files\n");
     DiB_shuffle(fileNamesTable, nbFiles);
+
     DiB_loadFiles(srcBuffer, &loadedSize, sampleSizes, fs.nbSamples, fileNamesTable, nbFiles, chunkSize, displayLevel);
 
     {   size_t dictSize;
@@ -320,19 +325,36 @@ int DiB_trainFromFiles(const char* dictFileName, unsigned maxDictSize,
             dictSize = ZDICT_trainFromBuffer_unsafe_legacy(dictBuffer, maxDictSize,
                                                            srcBuffer, sampleSizes, fs.nbSamples,
                                                            *params);
-        } else if (optimizeCover) {
-            assert(coverParams != NULL);
-            dictSize = ZDICT_optimizeTrainFromBuffer_cover(dictBuffer, maxDictSize,
-                                                           srcBuffer, sampleSizes, fs.nbSamples,
-                                                           coverParams);
-            if (!ZDICT_isError(dictSize)) {
-                unsigned splitPercentage = (unsigned)(coverParams->splitPoint * 100);
-                DISPLAYLEVEL(2, "k=%u\nd=%u\nsteps=%u\nsplit=%u\n", coverParams->k, coverParams->d, coverParams->steps, splitPercentage);
+        } else if (coverParams) {
+            if (optimize) {
+              dictSize = ZDICT_optimizeTrainFromBuffer_cover(dictBuffer, maxDictSize,
+                                                             srcBuffer, sampleSizes, fs.nbSamples,
+                                                             coverParams);
+              if (!ZDICT_isError(dictSize)) {
+                  unsigned splitPercentage = (unsigned)(coverParams->splitPoint * 100);
+                  DISPLAYLEVEL(2, "k=%u\nd=%u\nsteps=%u\nsplit=%u\n", coverParams->k, coverParams->d,
+                              coverParams->steps, splitPercentage);
+              }
+            } else {
+              dictSize = ZDICT_trainFromBuffer_cover(dictBuffer, maxDictSize, srcBuffer,
+                                                     sampleSizes, fs.nbSamples, *coverParams);
             }
         } else {
-            assert(coverParams != NULL);
-            dictSize = ZDICT_trainFromBuffer_cover(dictBuffer, maxDictSize, srcBuffer,
-                                                   sampleSizes, fs.nbSamples, *coverParams);
+            assert(fastCoverParams != NULL);
+            if (optimize) {
+              dictSize = ZDICT_optimizeTrainFromBuffer_fastCover(dictBuffer, maxDictSize,
+                                                              srcBuffer, sampleSizes, fs.nbSamples,
+                                                              fastCoverParams);
+              if (!ZDICT_isError(dictSize)) {
+                unsigned splitPercentage = (unsigned)(fastCoverParams->splitPoint * 100);
+                DISPLAYLEVEL(2, "k=%u\nd=%u\nf=%u\nsteps=%u\nsplit=%u\naccel=%u\n", fastCoverParams->k,
+                            fastCoverParams->d, fastCoverParams->f, fastCoverParams->steps, splitPercentage,
+                            fastCoverParams->accel);
+              }
+            } else {
+              dictSize = ZDICT_trainFromBuffer_fastCover(dictBuffer, maxDictSize, srcBuffer,
+                                                        sampleSizes, fs.nbSamples, *fastCoverParams);
+            }
         }
         if (ZDICT_isError(dictSize)) {
             DISPLAYLEVEL(1, "dictionary training failed : %s \n", ZDICT_getErrorName(dictSize));   /* should not happen */
diff --git a/programs/dibio.h b/programs/dibio.h
index 31a6b4bdb..ea163fe6a 100644
--- a/programs/dibio.h
+++ b/programs/dibio.h
@@ -34,6 +34,6 @@
 int DiB_trainFromFiles(const char* dictFileName, unsigned maxDictSize,
                        const char** fileNamesTable, unsigned nbFiles, size_t chunkSize,
                        ZDICT_legacy_params_t* params, ZDICT_cover_params_t* coverParams,
-                       int optimizeCover);
+                       ZDICT_fastCover_params_t* fastCoverParams, int optimize);
 
 #endif
diff --git a/programs/zstd.1 b/programs/zstd.1
index 58473102e..51825af74 100644
--- a/programs/zstd.1
+++ b/programs/zstd.1
@@ -194,7 +194,7 @@ All arguments after \fB\-\-\fR are treated as files
 Use FILEs as training set to create a dictionary\. The training set should contain a lot of small files (> 100), and weight typically 100x the target dictionary size (for example, 10 MB for a 100 KB dictionary)\.
 .
 .IP
-Supports multithreading if \fBzstd\fR is compiled with threading support\. Additional parameters can be specified with \fB\-\-train\-cover\fR\. The legacy dictionary builder can be accessed with \fB\-\-train\-legacy\fR\. Equivalent to \fB\-\-train\-cover=d=8,steps=4\fR\.
+Supports multithreading if \fBzstd\fR is compiled with threading support\. Additional parameters can be specified with \fB\-\-train\-cover\fR\ or \fB\-\-train\-fastcover\fR\. The legacy dictionary builder can be accessed with \fB\-\-train\-legacy\fR\. Equivalent to \fB\-\-train\-fastCover=d=8,steps=4\fR\.
 .
 .TP
 \fB\-o file\fR
@@ -240,6 +240,25 @@ Examples:
 \fBzstd \-\-train\-cover=k=50 FILEs\fR
 .
 .TP
+\fB\-\-train\-fastcover[=k#,d=#,f=#,steps=#,split=#,accel=#]\fR
+Same as cover but with extra parameters \fIf\fR and \fIaccel\fR and different default value of split
+.
+.IP
+If \fIsplit\fR is not specified, then it tries \fIsplit\fR = 75. If \fIf\fR is not specified, then it tries \fIf\fR = 18. Requires that 0 < \fIf\fR < 32. If \fIaccel\fR is not specified, then it tries \fIaccel\fR = 1. Requires that 0 < \fIaccel\fR <= 10. Requires that \fId\fR = 6 or \fId\fR = 8.
+.
+.IP
+\fIf\fR is log of size of array that keeps track of frequency of subsegments of size \fId\fR. The subsegment is hashed to an index in the range [0,2^\fIf\fR - 1]. It is possible that 2 different subsegments are hashed to the same index, and they are considered as the same subsegment when computing frequency. Using a higher \fIf\fR reduces collision but takes longer.
+.
+.IP
+Examples:
+.
+.IP
+\fBzstd \-\-train\-fastcover FILEs\fR
+.
+.IP
+\fBzstd \-\-train\-fastcover=d=8,f=15,accel=2 FILEs\fR
+.
+.TP
 \fB\-\-train\-legacy[=selectivity=#]\fR
 Use legacy dictionary builder algorithm with the given dictionary \fIselectivity\fR (default: 9)\. The smaller the \fIselectivity\fR value, the denser the dictionary, improving its efficiency but reducing its possible maximum size\. \fB\-\-train\-legacy=s=#\fR is also accepted\.
 .
diff --git a/programs/zstd.1.md b/programs/zstd.1.md
index 055c5c244..324b765b7 100644
--- a/programs/zstd.1.md
+++ b/programs/zstd.1.md
@@ -254,6 +254,26 @@ Compression of small files similar to the sample set will be greatly improved.
 
     `zstd --train-cover=k=50,split=60 FILEs`
 
+* `--train-fastcover[=k#,d=#,f=#,steps=#,split=#,accel=#]`:
+    Same as cover but with extra parameters _f_ and _accel_ and different default value of split
+    If _split_ is not specified, then it tries _split_ = 75.
+    If _f_ is not specified, then it tries _f_ = 18.
+    Requires that 0 < _f_ < 32.
+    If _accel_ is not specified, then it tries _accel_ = 1.
+    Requires that 0 < _accel_ <= 10.
+    Requires that _d_ = 6 or _d_ = 8.
+
+    _f_ is log of size of array that keeps track of frequency of subsegments of size _d_.
+    The subsegment is hashed to an index in the range [0,2^_f_ - 1].
+    It is possible that 2 different subsegments are hashed to the same index, and they are considered as the same subsegment when computing frequency.
+    Using a higher _f_ reduces collision but takes longer.
+
+    Examples:
+
+    `zstd --train-fastcover FILEs`
+
+    `zstd --train-fastcover=d=8,f=15,accel=2 FILEs`
+
 * `--train-legacy[=selectivity=#]`:
     Use legacy dictionary builder algorithm with the given dictionary
     _selectivity_ (default: 9).
diff --git a/programs/zstdcli.c b/programs/zstdcli.c
index d5a2216d6..f7c5934d9 100644
--- a/programs/zstdcli.c
+++ b/programs/zstdcli.c
@@ -84,7 +84,10 @@ static U32 g_ldmMinMatch = 0;
 static U32 g_ldmHashEveryLog = LDM_PARAM_DEFAULT;
 static U32 g_ldmBucketSizeLog = LDM_PARAM_DEFAULT;
 
-#define DEFAULT_SPLITPOINT 1.0
+
+#define DEFAULT_ACCEL 1
+
+typedef enum { cover, fastCover, legacy } dictType;
 
 /*-************************************
 *  Display Macros
@@ -172,6 +175,7 @@ static int usage_advanced(const char* programName)
     DISPLAY( "Dictionary builder : \n");
     DISPLAY( "--train ## : create a dictionary from a training set of files \n");
     DISPLAY( "--train-cover[=k=#,d=#,steps=#,split=#] : use the cover algorithm with optional args\n");
+    DISPLAY( "--train-fastcover[=k=#,d=#,f=#,steps=#,split=#,accel=#] : use the fast cover algorithm with optional args\n");
     DISPLAY( "--train-legacy[=s=#] : use the legacy algorithm with selectivity (default: %u)\n", g_defaultSelectivityLevel);
     DISPLAY( " -o file : `file` is dictionary name (default: %s) \n", g_defaultDictName);
     DISPLAY( "--maxdict=# : limit dictionary to specified size (default: %u) \n", g_defaultMaxDictSize);
@@ -295,6 +299,33 @@ static unsigned parseCoverParameters(const char* stringPtr, ZDICT_cover_params_t
     return 1;
 }
 
+/**
+ * parseFastCoverParameters() :
+ * reads fastcover parameters from *stringPtr (e.g. "--train-fastcover=k=48,d=8,f=20,steps=32,accel=2") into *params
+ * @return 1 means that fastcover parameters were correct
+ * @return 0 in case of malformed parameters
+ */
+static unsigned parseFastCoverParameters(const char* stringPtr, ZDICT_fastCover_params_t* params)
+{
+    memset(params, 0, sizeof(*params));
+    for (; ;) {
+        if (longCommandWArg(&stringPtr, "k=")) { params->k = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
+        if (longCommandWArg(&stringPtr, "d=")) { params->d = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
+        if (longCommandWArg(&stringPtr, "f=")) { params->f = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
+        if (longCommandWArg(&stringPtr, "steps=")) { params->steps = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
+        if (longCommandWArg(&stringPtr, "accel=")) { params->accel = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
+        if (longCommandWArg(&stringPtr, "split=")) {
+          unsigned splitPercentage = readU32FromChar(&stringPtr);
+          params->splitPoint = (double)splitPercentage / 100.0;
+          if (stringPtr[0]==',') { stringPtr++; continue; } else break;
+        }
+        return 0;
+    }
+    if (stringPtr[0] != 0) return 0;
+    DISPLAYLEVEL(4, "cover: k=%u\nd=%u\nf=%u\nsteps=%u\nsplit=%u\naccel=%u\n", params->k, params->d, params->f, params->steps, (unsigned)(params->splitPoint * 100), params->accel);
+    return 1;
+}
+
 /**
  * parseLegacyParameters() :
  * reads legacy dictioanry builter parameters from *stringPtr (e.g. "--train-legacy=selectivity=8") into *selectivity
@@ -316,7 +347,19 @@ static ZDICT_cover_params_t defaultCoverParams(void)
     memset(¶ms, 0, sizeof(params));
     params.d = 8;
     params.steps = 4;
-    params.splitPoint = DEFAULT_SPLITPOINT;
+    params.splitPoint = 1.0;
+    return params;
+}
+
+static ZDICT_fastCover_params_t defaultFastCoverParams(void)
+{
+    ZDICT_fastCover_params_t params;
+    memset(¶ms, 0, sizeof(params));
+    params.d = 8;
+    params.f = 18;
+    params.steps = 4;
+    params.splitPoint = 0.75; /* different from default splitPoint of cover */
+    params.accel = DEFAULT_ACCEL;
     return params;
 }
 #endif
@@ -431,7 +474,8 @@ int main(int argCount, const char* argv[])
 #endif
 #ifndef ZSTD_NODICT
     ZDICT_cover_params_t coverParams = defaultCoverParams();
-    int cover = 1;
+    ZDICT_fastCover_params_t fastCoverParams = defaultFastCoverParams();
+    dictType dict = fastCover;
 #endif
 #ifndef ZSTD_NOBENCH
     BMK_advancedParams_t benchParams = BMK_initAdvancedParams();
@@ -530,18 +574,29 @@ int main(int argCount, const char* argv[])
                       operation = zom_train;
                       if (outFileName == NULL)
                           outFileName = g_defaultDictName;
-                      cover = 1;
+                      dict = cover;
                       /* Allow optional arguments following an = */
                       if (*argument == 0) { memset(&coverParams, 0, sizeof(coverParams)); }
                       else if (*argument++ != '=') { CLEAN_RETURN(badusage(programName)); }
                       else if (!parseCoverParameters(argument, &coverParams)) { CLEAN_RETURN(badusage(programName)); }
                       continue;
                     }
+                    if (longCommandWArg(&argument, "--train-fastcover")) {
+                      operation = zom_train;
+                      if (outFileName == NULL)
+                          outFileName = g_defaultDictName;
+                      dict = fastCover;
+                      /* Allow optional arguments following an = */
+                      if (*argument == 0) { memset(&fastCoverParams, 0, sizeof(fastCoverParams)); }
+                      else if (*argument++ != '=') { CLEAN_RETURN(badusage(programName)); }
+                      else if (!parseFastCoverParameters(argument, &fastCoverParams)) { CLEAN_RETURN(badusage(programName)); }
+                      continue;
+                    }
                     if (longCommandWArg(&argument, "--train-legacy")) {
                       operation = zom_train;
                       if (outFileName == NULL)
                           outFileName = g_defaultDictName;
-                      cover = 0;
+                      dict = legacy;
                       /* Allow optional arguments following an = */
                       if (*argument == 0) { continue; }
                       else if (*argument++ != '=') { CLEAN_RETURN(badusage(programName)); }
@@ -881,17 +936,22 @@ int main(int argCount, const char* argv[])
         zParams.compressionLevel = dictCLevel;
         zParams.notificationLevel = g_displayLevel;
         zParams.dictID = dictID;
-        if (cover) {
+        if (dict == cover) {
             int const optimize = !coverParams.k || !coverParams.d;
             coverParams.nbThreads = nbWorkers;
             coverParams.zParams = zParams;
-            operationResult = DiB_trainFromFiles(outFileName, maxDictSize, filenameTable, filenameIdx, blockSize, NULL, &coverParams, optimize);
+            operationResult = DiB_trainFromFiles(outFileName, maxDictSize, filenameTable, filenameIdx, blockSize, NULL, &coverParams, NULL, optimize);
+        } else if (dict == fastCover) {
+            int const optimize = !fastCoverParams.k || !fastCoverParams.d;
+            fastCoverParams.nbThreads = nbWorkers;
+            fastCoverParams.zParams = zParams;
+            operationResult = DiB_trainFromFiles(outFileName, maxDictSize, filenameTable, filenameIdx, blockSize, NULL, NULL, &fastCoverParams, optimize);
         } else {
             ZDICT_legacy_params_t dictParams;
             memset(&dictParams, 0, sizeof(dictParams));
             dictParams.selectivityLevel = dictSelect;
             dictParams.zParams = zParams;
-            operationResult = DiB_trainFromFiles(outFileName, maxDictSize, filenameTable, filenameIdx, blockSize, &dictParams, NULL, 0);
+            operationResult = DiB_trainFromFiles(outFileName, maxDictSize, filenameTable, filenameIdx, blockSize, &dictParams, NULL, NULL, 0);
         }
 #endif
         goto _end;
diff --git a/tests/playTests.sh b/tests/playTests.sh
index 0a1f96c02..fa0685c6d 100755
--- a/tests/playTests.sh
+++ b/tests/playTests.sh
@@ -427,7 +427,7 @@ $ECHO "- Create second (different) dictionary"
 $ZSTD --train-cover=k=56,d=8 *.c ../programs/*.c ../programs/*.h -o tmpDictC
 $ZSTD -d tmp.zst -D tmpDictC -fo result && die "wrong dictionary not detected!"
 $ECHO "- Create dictionary with short dictID"
-$ZSTD --train-cover=k=46,d=8 *.c ../programs/*.c --dictID=1 -o tmpDict1
+$ZSTD --train-cover=k=46,d=8,split=80 *.c ../programs/*.c --dictID=1 -o tmpDict1
 cmp tmpDict tmpDict1 && die "dictionaries should have different ID !"
 $ECHO "- Create dictionary with size limit"
 $ZSTD --train-cover=steps=8 *.c ../programs/*.c -o tmpDict2 --maxdict=4K
@@ -444,6 +444,47 @@ $ZSTD --train-cover *.c ../programs/*.c
 test -f dictionary
 rm tmp* dictionary
 
+
+$ECHO "\n===>  fastCover dictionary builder : advanced options "
+
+TESTFILE=../programs/zstdcli.c
+./datagen > tmpDict
+$ECHO "- Create first dictionary"
+$ZSTD --train-fastcover=k=46,d=8,f=15,split=80 *.c ../programs/*.c -o tmpDict
+cp $TESTFILE tmp
+$ZSTD -f tmp -D tmpDict
+$ZSTD -d tmp.zst -D tmpDict -fo result
+$DIFF $TESTFILE result
+$ECHO "- Create second (different) dictionary"
+$ZSTD --train-fastcover=k=56,d=8 *.c ../programs/*.c ../programs/*.h -o tmpDictC
+$ZSTD -d tmp.zst -D tmpDictC -fo result && die "wrong dictionary not detected!"
+$ECHO "- Create dictionary with short dictID"
+$ZSTD --train-fastcover=k=46,d=8,f=15,split=80 *.c ../programs/*.c --dictID=1 -o tmpDict1
+cmp tmpDict tmpDict1 && die "dictionaries should have different ID !"
+$ECHO "- Create dictionary with size limit"
+$ZSTD --train-fastcover=steps=8 *.c ../programs/*.c -o tmpDict2 --maxdict=4K
+$ECHO "- Compare size of dictionary from 90% training samples with 80% training samples"
+$ZSTD --train-fastcover=split=90 -r *.c ../programs/*.c
+$ZSTD --train-fastcover=split=80 -r *.c ../programs/*.c
+$ECHO "- Create dictionary using all samples for both training and testing"
+$ZSTD --train-fastcover=split=100 -r *.c ../programs/*.c
+$ECHO "- Create dictionary using f=16"
+$ZSTD --train-fastcover=f=16 -r *.c ../programs/*.c
+$ECHO "- Create dictionary using accel=2"
+$ZSTD --train-fastcover=accel=2 -r *.c ../programs/*.c
+$ECHO "- Create dictionary using accel=10"
+$ZSTD --train-fastcover=accel=10 -r *.c ../programs/*.c
+$ECHO "- Create dictionary with multithreading"
+$ZSTD --train-fastcover -T4 -r *.c ../programs/*.c
+$ECHO "- Test -o before --train-fastcover"
+rm -f tmpDict dictionary
+$ZSTD -o tmpDict --train-fastcover *.c ../programs/*.c
+test -f tmpDict
+$ZSTD --train-fastcover *.c ../programs/*.c
+test -f dictionary
+rm tmp* dictionary
+
+
 $ECHO "\n===>  legacy dictionary builder "
 
 TESTFILE=../programs/zstdcli.c
diff --git a/tests/symbols.c b/tests/symbols.c
index c0bed2e5d..b37082131 100644
--- a/tests/symbols.c
+++ b/tests/symbols.c
@@ -144,6 +144,8 @@ static const void *symbols[] = {
 /* zdict.h: advanced functions */
   &ZDICT_trainFromBuffer_cover,
   &ZDICT_optimizeTrainFromBuffer_cover,
+  &ZDICT_trainFromBuffer_fastCover,
+  &ZDICT_optimizeTrainFromBuffer_fastCover,
   &ZDICT_finalizeDictionary,
   &ZDICT_trainFromBuffer_legacy,
   &ZDICT_addEntropyTablesFromBuffer,

From 2e45badff48a92842a8e9acc4c19589d272ebfc0 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Thu, 23 Aug 2018 14:21:18 -0700
Subject: [PATCH 196/372] refactored bench.c

for clarity and safety, especially at interface level
---
 lib/compress/zstd_compress.c |   3 +-
 programs/bench.c             | 893 +++++++++++++++++++----------------
 programs/bench.h             | 166 ++++---
 3 files changed, 587 insertions(+), 475 deletions(-)

diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index 29dce1250..48ccdeefa 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -1322,8 +1322,7 @@ static size_t ZSTD_resetCCtx_usingCDict(ZSTD_CCtx* cctx,
         }
 
         /* copy dictionary offsets */
-        {
-            ZSTD_matchState_t const* srcMatchState = &cdict->matchState;
+        {   ZSTD_matchState_t const* srcMatchState = &cdict->matchState;
             ZSTD_matchState_t* dstMatchState = &cctx->blockState.matchState;
             dstMatchState->window       = srcMatchState->window;
             dstMatchState->nextToUpdate = srcMatchState->nextToUpdate;
diff --git a/programs/bench.c b/programs/bench.c
index f280f00ef..d95adb554 100644
--- a/programs/bench.c
+++ b/programs/bench.c
@@ -63,7 +63,10 @@
 #define MB *(1 <<20)
 #define GB *(1U<<30)
 
-static const size_t maxMemory = (sizeof(size_t)==4)  ?  (2 GB - 64 MB) : (size_t)(1ULL << ((sizeof(size_t)*8)-31));
+static const size_t maxMemory = (sizeof(size_t)==4)  ?
+                    /* 32-bit */ (2 GB - 64 MB) :
+                    /* 64-bit */ (size_t)(1ULL << ((sizeof(size_t)*8)-31));
+
 
 /* *************************************
 *  console display
@@ -97,26 +100,26 @@ static UTIL_time_t g_displayClock = UTIL_TIME_INITIALIZER;
     return errorNum;                                  \
 }
 
-#define EXM_THROW(errorNum, retType, ...)  {          \
+#define RETURN_ERROR(errorNum, retType, ...)  {       \
     retType r;                                        \
     memset(&r, 0, sizeof(retType));                   \
     DEBUGOUTPUT("%s: %i: \n", __FILE__, __LINE__);    \
     DISPLAYLEVEL(1, "Error %i : ", errorNum);         \
     DISPLAYLEVEL(1, __VA_ARGS__);                     \
     DISPLAYLEVEL(1, " \n");                           \
-    r.error = errorNum;                               \
+    r.tag = errorNum;                                 \
     return r;                                         \
 }
 
 /* error without displaying */
-#define EXM_THROW_ND(errorNum, retType, ...)  {       \
+#define RETURN_QUIET_ERROR(errorNum, retType, ...)  { \
     retType r;                                        \
     memset(&r, 0, sizeof(retType));                   \
     DEBUGOUTPUT("%s: %i: \n", __FILE__, __LINE__);    \
     DEBUGOUTPUT("Error %i : ", errorNum);             \
     DEBUGOUTPUT(__VA_ARGS__);                         \
     DEBUGOUTPUT(" \n");                               \
-    r.error = errorNum;                               \
+    r.tag = errorNum;                                 \
     return r;                                         \
 }
 
@@ -125,9 +128,8 @@ static UTIL_time_t g_displayClock = UTIL_TIME_INITIALIZER;
 ***************************************/
 
 BMK_advancedParams_t BMK_initAdvancedParams(void) {
-    BMK_advancedParams_t res = {
+    BMK_advancedParams_t const res = {
         BMK_both, /* mode */
-        BMK_timeMode, /* loopMode */
         BMK_TIMETEST_DEFAULT_S, /* nbSeconds */
         0, /* blockSize */
         0, /* nbWorkers */
@@ -156,13 +158,6 @@ typedef struct {
     size_t resSize;
 } blockParam_t;
 
-struct  BMK_timeState_t{
-    unsigned nbLoops;
-    U64 timeRemaining;
-    UTIL_time_t coolTime;
-    U64 fastestTime;
-};
-
 #undef MIN
 #undef MAX
 #define MIN(a,b)    ((a) < (b) ? (a) : (b))
@@ -194,15 +189,15 @@ static void BMK_initCCtx(ZSTD_CCtx* ctx,
     ZSTD_CCtx_loadDictionary(ctx, dictBuffer, dictBufferSize);
 }
 
-
 static void BMK_initDCtx(ZSTD_DCtx* dctx,
     const void* dictBuffer, size_t dictBufferSize) {
     ZSTD_DCtx_reset(dctx);
     ZSTD_DCtx_loadDictionary(dctx, dictBuffer, dictBufferSize);
 }
 
+
 typedef struct {
-    ZSTD_CCtx* ctx;
+    ZSTD_CCtx* cctx;
     const void* dictBuffer;
     size_t dictBufferSize;
     int cLevel;
@@ -212,7 +207,7 @@ typedef struct {
 
 static size_t local_initCCtx(void* payload) {
     BMK_initCCtxArgs* ag = (BMK_initCCtxArgs*)payload;
-    BMK_initCCtx(ag->ctx, ag->dictBuffer, ag->dictBufferSize, ag->cLevel, ag->comprParams, ag->adv);
+    BMK_initCCtx(ag->cctx, ag->dictBuffer, ag->dictBufferSize, ag->cLevel, ag->comprParams, ag->adv);
     return 0;
 }
 
@@ -228,26 +223,24 @@ static size_t local_initDCtx(void* payload) {
     return 0;
 }
 
-/* additional argument is just the context */
+
+/* `addArgs` is the context */
 static size_t local_defaultCompress(
-    const void* srcBuffer, size_t srcSize,
-    void* dstBuffer, size_t dstSize,
-    void* addArgs) {
+                    const void* srcBuffer, size_t srcSize,
+                    void* dstBuffer, size_t dstSize,
+                    void* addArgs)
+{
     size_t moreToFlush = 1;
-    ZSTD_CCtx* ctx = (ZSTD_CCtx*)addArgs;
+    ZSTD_CCtx* const cctx = (ZSTD_CCtx*)addArgs;
     ZSTD_inBuffer in;
     ZSTD_outBuffer out;
-    in.src = srcBuffer;
-    in.size = srcSize;
-    in.pos = 0;
-    out.dst = dstBuffer;
-    out.size = dstSize;
-    out.pos = 0;
+    in.src = srcBuffer; in.size = srcSize; in.pos = 0;
+    out.dst = dstBuffer; out.size = dstSize; out.pos = 0;
     while (moreToFlush) {
         if(out.pos == out.size) {
             return (size_t)-ZSTD_error_dstSize_tooSmall;
         }
-        moreToFlush = ZSTD_compress_generic(ctx, &out, &in, ZSTD_e_end);
+        moreToFlush = ZSTD_compress_generic(cctx, &out, &in, ZSTD_e_end);
         if (ZSTD_isError(moreToFlush)) {
             return moreToFlush;
         }
@@ -255,27 +248,23 @@ static size_t local_defaultCompress(
     return out.pos;
 }
 
-/* additional argument is just the context */
+/* `addArgs` is the context */
 static size_t local_defaultDecompress(
-    const void* srcBuffer, size_t srcSize,
-    void* dstBuffer, size_t dstSize,
-    void* addArgs) {
+                    const void* srcBuffer, size_t srcSize,
+                    void* dstBuffer, size_t dstSize,
+                    void* addArgs)
+{
     size_t moreToFlush = 1;
-    ZSTD_DCtx* dctx = (ZSTD_DCtx*)addArgs;
+    ZSTD_DCtx* const dctx = (ZSTD_DCtx*)addArgs;
     ZSTD_inBuffer in;
     ZSTD_outBuffer out;
-    in.src = srcBuffer;
-    in.size = srcSize;
-    in.pos = 0;
-    out.dst = dstBuffer;
-    out.size = dstSize;
-    out.pos = 0;
+    in.src = srcBuffer; in.size = srcSize; in.pos = 0;
+    out.dst = dstBuffer; out.size = dstSize; out.pos = 0;
     while (moreToFlush) {
         if(out.pos == out.size) {
             return (size_t)-ZSTD_error_dstSize_tooSmall;
         }
-        moreToFlush = ZSTD_decompress_generic(dctx,
-                            &out, &in);
+        moreToFlush = ZSTD_decompress_generic(dctx, &out, &in);
         if (ZSTD_isError(moreToFlush)) {
             return moreToFlush;
         }
@@ -284,30 +273,56 @@ static size_t local_defaultDecompress(
 
 }
 
-/* initFn will be measured once, bench fn will be measured x times */
-/* benchFn should return error value or out Size */
+
+/*===  Benchmarking an arbitrary function  ===*/
+
+int BMK_isSuccessful_runOutcome(BMK_runOutcome_t outcome)
+{
+    return outcome.tag == 0;
+}
+
+/* warning : this function will stop program execution if outcome is invalid !
+ *           check outcome validity first, using BMK_isValid_runResult() */
+BMK_runTime_t BMK_extract_runTime(BMK_runOutcome_t outcome)
+{
+    assert(outcome.tag == 0);
+    return outcome.internal_never_use_directly;
+}
+
+static BMK_runOutcome_t BMK_setValid_runTime(BMK_runTime_t runTime)
+{
+    BMK_runOutcome_t outcome;
+    outcome.tag = 0;
+    outcome.internal_never_use_directly = runTime;
+    return outcome;
+}
+
+
+/* initFn will be measured once, benchFn will be measured `nbLoops` times */
+/* initFn is optional, provide NULL if none */
+/* benchFn must return size_t field compliant with ZSTD_isError for error valuee */
 /* takes # of blocks and list of size & stuff for each. */
-/* only does looping */
-/* note time per loop could be zero if interval too short */
-BMK_customReturn_t BMK_benchFunction(
-    BMK_benchFn_t benchFn, void* benchPayload,
-    BMK_initFn_t initFn, void* initPayload,
-    size_t blockCount,
-    const void* const * const srcBlockBuffers, const size_t* srcBlockSizes,
-    void* const * const dstBlockBuffers, const size_t* dstBlockCapacities, size_t* blockResult,
-    unsigned nbLoops) {
+/* can report result of benchFn for each block into blockResult. */
+/* blockResult is optional, provide NULL if this information is not required */
+/* note : time per loop could be zero if run time < timer resolution */
+BMK_runOutcome_t BMK_benchFunction(
+            BMK_benchFn_t benchFn, void* benchPayload,
+            BMK_initFn_t initFn, void* initPayload,
+            size_t blockCount,
+            const void* const * srcBlockBuffers, const size_t* srcBlockSizes,
+            void* const * dstBlockBuffers, const size_t* dstBlockCapacities,
+            size_t* blockResult,
+            unsigned nbLoops)
+{
     size_t dstSize = 0;
     U64 totalTime;
 
-    BMK_customReturn_t retval;
-    UTIL_time_t clockStart;
-
     if(!nbLoops) {
-        EXM_THROW_ND(1, BMK_customReturn_t, "nbLoops must be nonzero \n");
+        RETURN_QUIET_ERROR(1, BMK_runOutcome_t, "nbLoops must be nonzero ");
     }
 
-    {
-        size_t i;
+    /* init */
+    {   size_t i;
         for(i = 0; i < blockCount; i++) {
             memset(dstBlockBuffers[i], 0xE5, dstBlockCapacities[i]);  /* warm up and erase result buffer */
         }
@@ -320,157 +335,247 @@ BMK_customReturn_t BMK_benchFunction(
 #endif
     }
 
-    {
-        unsigned i, j;
-        clockStart = UTIL_getTime();
-        if(initFn != NULL) { initFn(initPayload); }
-        for(i = 0; i < nbLoops; i++) {
-            for(j = 0; j < blockCount; j++) {
-                size_t res = benchFn(srcBlockBuffers[j], srcBlockSizes[j], dstBlockBuffers[j], dstBlockCapacities[j], benchPayload);
+    /* benchmark loop */
+    {   UTIL_time_t const clockStart = UTIL_getTime();
+        unsigned loopNb, blockNb;
+        if (initFn != NULL) initFn(initPayload);
+        for (loopNb = 0; loopNb < nbLoops; loopNb++) {
+            for (blockNb = 0; blockNb < blockCount; blockNb++) {
+                size_t const res = benchFn(srcBlockBuffers[blockNb], srcBlockSizes[blockNb],
+                                    dstBlockBuffers[blockNb], dstBlockCapacities[blockNb],
+                                    benchPayload);
                 if(ZSTD_isError(res)) {
-                    EXM_THROW_ND(2, BMK_customReturn_t, "Function benchmarking failed on block %u of size %u : %s  \n",
-                        j, (U32)dstBlockCapacities[j], ZSTD_getErrorName(res));
-                } else if(i == nbLoops - 1) {
+                    RETURN_QUIET_ERROR(2, BMK_runOutcome_t,
+                        "Function benchmark failed on block %u of size %u : %s",
+                        blockNb, (U32)dstBlockCapacities[blockNb], ZSTD_getErrorName(res));
+                } else if (loopNb == 0) {
                     dstSize += res;
-                    if(blockResult != NULL) {
-                        blockResult[j] = res;
-                    }
-                }
-            }
-        }
+                    if (blockResult != NULL) blockResult[blockNb] = res;
+                    dstSize += res;
+            }   }
+        }  /* for (loopNb = 0; loopNb < nbLoops; loopNb++) */
         totalTime = UTIL_clockSpanNano(clockStart);
     }
 
-    retval.error = 0;
-    retval.result.nanoSecPerRun = totalTime / nbLoops;
-    retval.result.sumOfReturn = dstSize;
-    return retval;
+    {   BMK_runTime_t rt;
+        rt.nanoSecPerRun = totalTime / nbLoops;
+        rt.sumOfReturn = dstSize;
+        return BMK_setValid_runTime(rt);
+    }
 }
 
-#define MINUSABLETIME 500000000ULL /* 0.5 seconds in ns */
 
-void BMK_resetTimedFnState(BMK_timedFnState_t* r, unsigned nbSeconds) {
-    r->nbLoops = 1;
-    r->timeRemaining = (U64)nbSeconds * TIMELOOP_NANOSEC;
-    r->coolTime = UTIL_getTime();
-    r->fastestTime = (U64)(-1LL);
-}
+/* ====  Benchmarking any function, providing intermediate results  ==== */
+
+struct BMK_timedFnState_s {
+    U64 timeSpent_ns;
+    U64 timeBudget_ns;
+    BMK_runTime_t fastestRun;
+    unsigned nbLoops;
+    UTIL_time_t coolTime;
+};  /* typedef'd to BMK_timedFnState_t within bench.h */
 
 BMK_timedFnState_t* BMK_createTimedFnState(unsigned nbSeconds) {
-    BMK_timedFnState_t* r = (BMK_timedFnState_t*)malloc(sizeof(struct BMK_timeState_t));
-    if(r == NULL) {
-        return r;
-    }
+    BMK_timedFnState_t* const r = (BMK_timedFnState_t*)malloc(sizeof(*r));
+    if (r == NULL) return NULL;   /* malloc() error */
     BMK_resetTimedFnState(r, nbSeconds);
     return r;
 }
 
+void BMK_resetTimedFnState(BMK_timedFnState_t* r, unsigned nbSeconds) {
+    r->timeSpent_ns = 0;
+    r->timeBudget_ns = (U64)nbSeconds * TIMELOOP_NANOSEC;
+    r->fastestRun.nanoSecPerRun = (U64)(-1LL);
+    r->fastestRun.sumOfReturn = (size_t)(-1LL);
+    r->nbLoops = 1;
+    r->coolTime = UTIL_getTime();
+}
+
 void BMK_freeTimedFnState(BMK_timedFnState_t* state) {
     free(state);
 }
 
-/* make option for dstBlocks to be */
-BMK_customTimedReturn_t BMK_benchFunctionTimed(
-    BMK_timedFnState_t* cont,
-    BMK_benchFn_t benchFn, void* benchPayload,
-    BMK_initFn_t initFn, void* initPayload,
-    size_t blockCount,
-    const void* const* const srcBlockBuffers, const size_t* srcBlockSizes,
-    void * const * const dstBlockBuffers, const size_t * dstBlockCapacities, size_t* blockResults)
-{
-    U64 fastest = cont->fastestTime;
-    int completed = 0;
-    BMK_customTimedReturn_t r;
-    r.completed = 0;
 
-    while(!r.completed && !completed)
-    {
+/* check first if the return structure represents an error or a valid result */
+int BMK_isSuccessful_timedFnOutcome(BMK_timedFnOutcome_t outcome)
+{
+    return (outcome.tag < 2);
+}
+
+/* extract intermediate results from variant type.
+ * note : this function will abort() program execution if result is not valid.
+ *        check result validity first, by using BMK_isSuccessful_timedFnOutcome() */
+BMK_runTime_t BMK_extract_timedFnResult(BMK_timedFnOutcome_t outcome)
+{
+    assert(outcome.tag < 2);
+    return outcome.internal_never_use_directly;
+}
+
+/* Tells if nb of seconds set in timedFnState for all runs is spent.
+ * note : this function will return 1 if BMK_benchFunctionTimed() has actually errored. */
+int BMK_isCompleted_timedFnOutcome(BMK_timedFnOutcome_t outcome)
+{
+    return (outcome.tag >= 1);
+}
+
+
+#define MINUSABLETIME  (TIMELOOP_NANOSEC / 2)  /* 0.5 seconds */
+
+BMK_timedFnOutcome_t BMK_benchFunctionTimed(
+            BMK_timedFnState_t* cont,
+            BMK_benchFn_t benchFn, void* benchPayload,
+            BMK_initFn_t initFn, void* initPayload,
+            size_t blockCount,
+            const void* const* srcBlockBuffers, const size_t* srcBlockSizes,
+            void * const * dstBlockBuffers, const size_t * dstBlockCapacities,
+            size_t* blockResults)
+{
+    int completed = 0;
+    BMK_timedFnOutcome_t r;
+    BMK_runTime_t bestRunTime = cont->fastestRun;
+
+    r.tag = 2;  /* error by default */
+
+    while (!completed) {
+        BMK_runOutcome_t runResult;
+
         /* Overheat protection */
         if (UTIL_clockSpanMicro(cont->coolTime) > ACTIVEPERIOD_MICROSEC) {
             DEBUGOUTPUT("\rcooling down ...    \r");
             UTIL_sleep(COOLPERIOD_SEC);
             cont->coolTime = UTIL_getTime();
         }
+
         /* reinitialize capacity */
-        r.result = BMK_benchFunction(benchFn, benchPayload, initFn, initPayload,
-        blockCount, srcBlockBuffers, srcBlockSizes, dstBlockBuffers, dstBlockCapacities, blockResults, cont->nbLoops);
-        if(r.result.error) { /* completed w/ error */
-            r.completed = 1;
+        runResult = BMK_benchFunction(benchFn, benchPayload,
+                                    initFn, initPayload,
+                                    blockCount,
+                                    srcBlockBuffers, srcBlockSizes,
+                                    dstBlockBuffers, dstBlockCapacities,
+                                    blockResults,
+                                    cont->nbLoops);
+
+        if(!BMK_isSuccessful_runOutcome(runResult)) { /* error : move out */
+            r.tag = 2;
             return r;
         }
 
-        {   U64 const loopDuration = r.result.result.nanoSecPerRun * cont->nbLoops;
-            r.completed = (cont->timeRemaining <= loopDuration);
-            cont->timeRemaining -= loopDuration;
-            if (loopDuration > (TIMELOOP_NANOSEC / 100)) {
-                fastest = MIN(fastest, r.result.result.nanoSecPerRun);
-                if(loopDuration >= MINUSABLETIME) {
-                    r.result.result.nanoSecPerRun = fastest;
-                    cont->fastestTime = fastest;
-                }
-                cont->nbLoops = (U32)(TIMELOOP_NANOSEC / r.result.result.nanoSecPerRun) + 1;
+        {   BMK_runTime_t const newRunTime = BMK_extract_runTime(runResult);
+            U64 const loopDuration_ns = newRunTime.nanoSecPerRun * cont->nbLoops;
+
+            cont->timeSpent_ns += loopDuration_ns;
+
+            /* estimate nbLoops for next run to last approximately 1 second */
+            if (loopDuration_ns > (TIMELOOP_NANOSEC / 50)) {
+                U64 const fastestRun_ns = MIN(bestRunTime.nanoSecPerRun, newRunTime.nanoSecPerRun);
+                cont->nbLoops = (U32)(TIMELOOP_NANOSEC / fastestRun_ns) + 1;
             } else {
-                const unsigned multiplier = 2;
+                /* previous run was too short : blindly increase workload by x multiplier */
+                const unsigned multiplier = 10;
                 assert(cont->nbLoops < ((unsigned)-1) / multiplier);  /* avoid overflow */
                 cont->nbLoops *= multiplier;
             }
-            if(loopDuration < MINUSABLETIME) { /* don't report results which have time too low */
-                continue;
-            }
 
+            if(loopDuration_ns < MINUSABLETIME) {
+                /* don't report results for which benchmark run time was too small : increased risks of rounding errors */
+                assert(completed == 0);
+                continue;
+            } else {
+                if(newRunTime.nanoSecPerRun < bestRunTime.nanoSecPerRun) {
+                    bestRunTime = newRunTime;
+                }
+                completed = 1;
+            }
         }
-        completed = 1;
-    }
+    }   /* while (!completed) */
+
+    r.tag = (cont->timeSpent_ns >= cont->timeBudget_ns);  /* report if time budget is spent */
+    r.internal_never_use_directly = bestRunTime;
     return r;
 }
 
-/* benchMem with no allocation */
-static BMK_return_t BMK_benchMemAdvancedNoAlloc(
-    const void ** const srcPtrs, size_t* const srcSizes,
-    void** const cPtrs, size_t* const cCapacities, size_t* const cSizes,
-    void** const resPtrs, size_t* const resSizes,
-    void** resultBufferPtr, void* compressedBuffer,
-    const size_t maxCompressedSize,
-    BMK_timedFnState_t* timeStateCompress, BMK_timedFnState_t* timeStateDecompress,
 
-    const void* srcBuffer, size_t srcSize,
-    const size_t* fileSizes, unsigned nbFiles,
-    const int cLevel, const ZSTD_compressionParameters* comprParams,
-    const void* dictBuffer, size_t dictBufferSize,
-    ZSTD_CCtx* ctx, ZSTD_DCtx* dctx,
-    int displayLevel, const char* displayName, const BMK_advancedParams_t* adv)
+/* ================================================================= */
+/*      Benchmark Zstandard, mem-to-mem scenarios                    */
+/* ================================================================= */
+
+int BMK_isSuccessful_benchOutcome(BMK_benchOutcome_t outcome)
+{
+    return outcome.tag == 0;
+}
+
+BMK_benchResult_t BMK_extract_benchResult(BMK_benchOutcome_t outcome)
+{
+    assert(outcome.tag == 0);
+    return outcome.internal_never_use_directly;
+}
+
+static BMK_benchOutcome_t BMK_benchOutcome_error()
+{
+    BMK_benchOutcome_t b;
+    memset(&b, 0, sizeof(b));
+    b.tag = 1;
+    return b;
+}
+
+static BMK_benchOutcome_t BMK_benchOutcome_setValidResult(BMK_benchResult_t result)
+{
+    BMK_benchOutcome_t b;
+    b.tag = 0;
+    b.internal_never_use_directly = result;
+    return b;
+}
+
+
+/* benchMem with no allocation */
+static BMK_benchOutcome_t BMK_benchMemAdvancedNoAlloc(
+            const void** srcPtrs, size_t* srcSizes,
+            void** cPtrs, size_t* cCapacities, size_t* cSizes,
+            void** resPtrs, size_t* resSizes,
+            void** resultBufferPtr, void* compressedBuffer,
+            size_t maxCompressedSize,
+            BMK_timedFnState_t* timeStateCompress,
+            BMK_timedFnState_t* timeStateDecompress,
+
+            const void* srcBuffer, size_t srcSize,
+            const size_t* fileSizes, unsigned nbFiles,
+            const int cLevel, const ZSTD_compressionParameters* comprParams,
+            const void* dictBuffer, size_t dictBufferSize,
+            ZSTD_CCtx* cctx, ZSTD_DCtx* dctx,
+            int displayLevel, const char* displayName,
+            const BMK_advancedParams_t* adv)
 {
     size_t const blockSize = ((adv->blockSize>=32 && (adv->mode != BMK_decodeOnly)) ? adv->blockSize : srcSize) + (!srcSize); /* avoid div by 0 */
-    BMK_return_t results = { { 0, 0, 0, 0 }, 0 } ;
+    BMK_benchResult_t benchResult;
     size_t const loadedCompressedSize = srcSize;
     size_t cSize = 0;
     double ratio = 0.;
     U32 nbBlocks;
 
-    if(!ctx || !dctx)
-        EXM_THROW(31, BMK_return_t, "error: passed in null context");
+    assert(cctx != NULL); assert(dctx != NULL);
 
     /* init */
-    if (strlen(displayName)>17) displayName += strlen(displayName)-17;   /* display last 17 characters */
+    if (strlen(displayName)>17) displayName += strlen(displayName) - 17;   /* display last 17 characters */
     if (adv->mode == BMK_decodeOnly) {  /* benchmark only decompression : source must be already compressed */
         const char* srcPtr = (const char*)srcBuffer;
         U64 totalDSize64 = 0;
         U32 fileNb;
         for (fileNb=0; fileNb decodedSize) {
+            if (totalDSize64 > decodedSize) {  /* size_t overflow */
                 free(*resultBufferPtr);
-                EXM_THROW(32, BMK_return_t, "original size is too large");   /* size_t overflow */
+                RETURN_ERROR(32, BMK_benchOutcome_t, "original size is too large");
             }
             cSize = srcSize;
             srcSize = decodedSize;
@@ -489,11 +594,11 @@ static BMK_return_t BMK_benchMemAdvancedNoAlloc(
             U32 const blockEnd = nbBlocks + nbBlocksforThisFile;
             for ( ; nbBlocksmode == BMK_decodeOnly) ? thisBlockSize : ZSTD_compressBound(thisBlockSize);
-                resPtrs[nbBlocks] = (void*)resPtr;
+                resPtrs[nbBlocks] = resPtr;
                 resSizes[nbBlocks] = (adv->mode == BMK_decodeOnly) ? (size_t) ZSTD_findDecompressedSize(srcPtr, thisBlockSize) : thisBlockSize;
                 srcPtr += thisBlockSize;
                 cPtr += cCapacities[nbBlocks];
@@ -503,7 +608,7 @@ static BMK_return_t BMK_benchMemAdvancedNoAlloc(
         }
     }
 
-    /* warmimg up memory */
+    /* warmimg up `compressedBuffer` */
     if (adv->mode == BMK_decodeOnly) {
         memcpy(compressedBuffer, srcBuffer, loadedCompressedSize);
     } else {
@@ -511,151 +616,99 @@ static BMK_return_t BMK_benchMemAdvancedNoAlloc(
     }
 
     /* Bench */
-    {
-        U64 const crcOrig = (adv->mode == BMK_decodeOnly) ? 0 : XXH64(srcBuffer, srcSize, 0);
+    {   U64 const crcOrig = (adv->mode == BMK_decodeOnly) ? 0 : XXH64(srcBuffer, srcSize, 0);
 #       define NB_MARKS 4
-        const char* const marks[NB_MARKS] = { " |", " /", " =",  "\\" };
+        const char* marks[NB_MARKS] = { " |", " /", " =", " \\" };
         U32 markNb = 0;
-        DISPLAYLEVEL(2, "\r%79s\r", "");
+        int compressionCompleted = (adv->mode == BMK_decodeOnly);
+        int decompressionCompleted = (adv->mode == BMK_compressOnly);
+        BMK_initCCtxArgs cctxprep;
+        BMK_initDCtxArgs dctxprep;
+        cctxprep.cctx = cctx;
+        cctxprep.dictBuffer = dictBuffer;
+        cctxprep.dictBufferSize = dictBufferSize;
+        cctxprep.cLevel = cLevel;
+        cctxprep.comprParams = comprParams;
+        cctxprep.adv = adv;
+        dctxprep.dctx = dctx;
+        dctxprep.dictBuffer = dictBuffer;
+        dctxprep.dictBufferSize = dictBufferSize;
 
+        DISPLAYLEVEL(2, "\r%70s\r", "");   /* blank line */
         DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->\r", marks[markNb], displayName, (U32)srcSize);
-        {
-            BMK_initCCtxArgs cctxprep;
-            BMK_initDCtxArgs dctxprep;
-            cctxprep.ctx = ctx;
-            cctxprep.dictBuffer = dictBuffer;
-            cctxprep.dictBufferSize = dictBufferSize;
-            cctxprep.cLevel = cLevel;
-            cctxprep.comprParams = comprParams;
-            cctxprep.adv = adv;
-            dctxprep.dctx = dctx;
-            dctxprep.dictBuffer = dictBuffer;
-            dctxprep.dictBufferSize = dictBufferSize;
-            if(adv->loopMode == BMK_timeMode) {
-                BMK_customTimedReturn_t intermediateResultCompress;
-                BMK_customTimedReturn_t intermediateResultDecompress;
-                if(adv->mode == BMK_compressOnly) {
-                    intermediateResultCompress.completed = 0;
-                    intermediateResultDecompress.completed = 1;
-                } else if (adv->mode == BMK_decodeOnly) {
-                    intermediateResultCompress.completed = 1;
-                    intermediateResultDecompress.completed = 0;
-                } else { /* both */
-                    intermediateResultCompress.completed = 0;
-                    intermediateResultDecompress.completed = 0;
-                }
-                while(!(intermediateResultCompress.completed && intermediateResultDecompress.completed)) {
-                    if(!intermediateResultCompress.completed) {
-                        intermediateResultCompress = BMK_benchFunctionTimed(timeStateCompress, &local_defaultCompress, (void*)ctx, &local_initCCtx, (void*)&cctxprep,
-                        nbBlocks, srcPtrs, srcSizes, cPtrs, cCapacities, cSizes);
-                        if(intermediateResultCompress.result.error) {
-                            results.error = intermediateResultCompress.result.error;
-                            return results;
-                        }
-                        ratio = (double)(srcSize / intermediateResultCompress.result.result.sumOfReturn);
-                        {
-                            int const ratioAccuracy = (ratio < 10.) ? 3 : 2;
-                            results.result.cSpeed = (srcSize * TIMELOOP_NANOSEC / intermediateResultCompress.result.result.nanoSecPerRun);
-                            cSize = intermediateResultCompress.result.result.sumOfReturn;
-                            results.result.cSize = cSize;
-                            ratio = (double)srcSize / results.result.cSize;
-                            markNb = (markNb+1) % NB_MARKS;
-                            DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.*f),%6.*f MB/s\r",
-                                    marks[markNb], displayName, (U32)srcSize, (U32)results.result.cSize,
-                                    ratioAccuracy, ratio,
-                                    results.result.cSpeed < (10 MB) ? 2 : 1, (double)results.result.cSpeed / (1 MB));
-                        }
-                    }
 
-                    if(!intermediateResultDecompress.completed) {
-                        intermediateResultDecompress = BMK_benchFunctionTimed(timeStateDecompress, &local_defaultDecompress, (void*)(dctx), &local_initDCtx, (void*)&dctxprep,
-                        nbBlocks, (const void* const*)cPtrs, cSizes, resPtrs, resSizes, NULL);
-                        if(intermediateResultDecompress.result.error) {
-                            results.error = intermediateResultDecompress.result.error;
-                            return results;
-                        }
+        while (!(compressionCompleted && decompressionCompleted)) {
 
-                        {
-                            int const ratioAccuracy = (ratio < 10.) ? 3 : 2;
-                            results.result.dSpeed = (srcSize * TIMELOOP_NANOSEC/ intermediateResultDecompress.result.result.nanoSecPerRun);
-                            markNb = (markNb+1) % NB_MARKS;
-                            DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.*f),%6.*f MB/s ,%6.1f MB/s \r",
-                                    marks[markNb], displayName, (U32)srcSize, (U32)results.result.cSize,
-                                    ratioAccuracy, ratio,
-                                    results.result.cSpeed < (10 MB) ? 2 : 1, (double)results.result.cSpeed / (1 MB),
-                                    (double)results.result.dSpeed / (1 MB));
-                        }
-                    }
+            if (!compressionCompleted) {
+                BMK_runTime_t cResult;
+
+                BMK_timedFnOutcome_t const cOutcome =
+                        BMK_benchFunctionTimed(timeStateCompress,
+                                    &local_defaultCompress, (void*)cctx,
+                                    &local_initCCtx, (void*)&cctxprep,
+                                    nbBlocks,
+                                    srcPtrs, srcSizes,
+                                    cPtrs, cCapacities,
+                                    cSizes);
+
+                if (!BMK_isSuccessful_timedFnOutcome(cOutcome)) {
+                    return BMK_benchOutcome_error();
                 }
 
-            } else { //iterMode;
-                if(adv->mode != BMK_decodeOnly) {
+                cResult = BMK_extract_timedFnResult(cOutcome);
+                ratio = (double)(srcSize / cResult.sumOfReturn);
 
-                    BMK_customReturn_t compressionResults = BMK_benchFunction(&local_defaultCompress, (void*)ctx, &local_initCCtx, (void*)&cctxprep,
-                        nbBlocks, srcPtrs, srcSizes, cPtrs, cCapacities, cSizes, adv->nbSeconds);
-                    if(compressionResults.error) {
-                        results.error = compressionResults.error;
-                        return results;
-                    }
-
-                    if(compressionResults.result.nanoSecPerRun == 0) {
-                        results.result.cSpeed = 0;
-                    } else {
-                        results.result.cSpeed = srcSize * TIMELOOP_NANOSEC / compressionResults.result.nanoSecPerRun;
-                    }
-
-                    results.result.cSize = compressionResults.result.sumOfReturn;
-                    {
-                        int const ratioAccuracy = (ratio < 10.) ? 3 : 2;
-                        cSize = compressionResults.result.sumOfReturn;
-                        results.result.cSize = cSize;
-                        ratio = (double)srcSize / results.result.cSize;
-                        markNb = (markNb+1) % NB_MARKS;
-                        DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.*f),%6.*f MB/s\r",
-                                marks[markNb], displayName, (U32)srcSize, (U32)results.result.cSize,
-                                ratioAccuracy, ratio,
-                                results.result.cSpeed < (10 MB) ? 2 : 1, (double)results.result.cSpeed / (1 MB));
-                    }
+                {   int const ratioAccuracy = (ratio < 10.) ? 3 : 2;
+                    cSize = cResult.sumOfReturn;
+                    benchResult.cSpeed = (srcSize * TIMELOOP_NANOSEC / cResult.nanoSecPerRun);
+                    benchResult.cSize = cSize;
+                    ratio = (double)srcSize / cSize;
+                    markNb = (markNb+1) % NB_MARKS;
+                    DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.*f),%6.*f MB/s\r",
+                            marks[markNb], displayName, (U32)srcSize, (U32)cSize,
+                            ratioAccuracy, ratio,
+                            benchResult.cSpeed < (10 MB) ? 2 : 1, (double)benchResult.cSpeed / (1 MB));
                 }
-                if(adv->mode != BMK_compressOnly) {
-                    BMK_customReturn_t decompressionResults = BMK_benchFunction(
-                        &local_defaultDecompress, (void*)(dctx),
-                        &local_initDCtx, (void*)&dctxprep, nbBlocks,
-                        (const void* const*)cPtrs, cSizes, resPtrs, resSizes, NULL,
-                        adv->nbSeconds);
-                    if(decompressionResults.error) {
-                        results.error = decompressionResults.error;
-                        return results;
-                    }
+            }
 
-                    if(decompressionResults.result.nanoSecPerRun == 0) {
-                        results.result.dSpeed = 0;
-                    } else {
-                        results.result.dSpeed = srcSize * TIMELOOP_NANOSEC / decompressionResults.result.nanoSecPerRun;
-                    }
+            if(!decompressionCompleted) {
+                BMK_runTime_t dResult;
 
-                    {
-                        int const ratioAccuracy = (ratio < 10.) ? 3 : 2;
-                        markNb = (markNb+1) % NB_MARKS;
-                        DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.*f),%6.*f MB/s ,%6.1f MB/s \r",
-                                marks[markNb], displayName, (U32)srcSize, (U32)results.result.cSize,
-                                ratioAccuracy, ratio,
-                                results.result.cSpeed < (10 MB) ? 2 : 1, (double)results.result.cSpeed / (1 MB),
-                                (double)results.result.dSpeed / (1 MB));
-                    }
+                BMK_timedFnOutcome_t const dOutcome =
+                            BMK_benchFunctionTimed(timeStateDecompress,
+                                    &local_defaultDecompress, (void*)(dctx),
+                                    &local_initDCtx, (void*)&dctxprep,
+                                    nbBlocks,
+                                    (const void* const*)cPtrs, cSizes,
+                                    resPtrs, resSizes,
+                                    NULL);
+
+                if(!BMK_isSuccessful_timedFnOutcome(dOutcome)) {
+                    return BMK_benchOutcome_error();
+                }
+
+                dResult = BMK_extract_timedFnResult(dOutcome);
+
+                {   int const ratioAccuracy = (ratio < 10.) ? 3 : 2;
+                    benchResult.dSpeed = (srcSize * TIMELOOP_NANOSEC / dResult.nanoSecPerRun);
+                    markNb = (markNb+1) % NB_MARKS;
+                    DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.*f),%6.*f MB/s ,%6.1f MB/s \r",
+                            marks[markNb], displayName, (U32)srcSize, (U32)benchResult.cSize,
+                            ratioAccuracy, ratio,
+                            benchResult.cSpeed < (10 MB) ? 2 : 1, (double)benchResult.cSpeed / (1 MB),
+                            (double)benchResult.dSpeed / (1 MB));
                 }
             }
         }
 
         /* CRC Checking */
-        {   void* resultBuffer = *resultBufferPtr;
+        {   const BYTE* resultBuffer = (const BYTE*)(*resultBufferPtr);
             U64 const crcCheck = XXH64(resultBuffer, srcSize, 0);
-            /* adv->mode == 0 -> compress + decompress */
             if ((adv->mode == BMK_both) && (crcOrig!=crcCheck)) {
                 size_t u;
                 DISPLAY("!!! WARNING !!! %14s : Invalid Checksum : %x != %x   \n", displayName, (unsigned)crcOrig, (unsigned)crcCheck);
                 for (u=0; u
  When compressing multiple messages / blocks with the same dictionary, it's recommended to load it just once.
   ZSTD_createCDict() will create a digested dictionary, ready to start future compression operations without startup delay.
   ZSTD_CDict can be created once and shared by multiple threads concurrently, since its usage is read-only.
-  `dictBuffer` can be released after ZSTD_CDict creation, since its content is copied within CDict 
+  `dictBuffer` can be released after ZSTD_CDict creation, since its content is copied within CDict
+  Note : A ZSTD_CDict can be created with an empty dictionary, but it is inefficient for small data. 
 


 
 
size_t      ZSTD_freeCDict(ZSTD_CDict* CDict);
@@ -195,7 +196,9 @@ size_t     ZSTD_freeDCtx(ZSTD_DCtx* dctx);
 
  Compression using a digested Dictionary.
   Faster startup than ZSTD_compress_usingDict(), recommended when same dictionary is used multiple times.
   Note that compression level is decided during dictionary creation.
-  Frame parameters are hardcoded (dictID=yes, contentSize=yes, checksum=no) 
+  Frame parameters are hardcoded (dictID=yes, contentSize=yes, checksum=no)
+  Note : ZSTD_compress_usingCDict() can be used with a ZSTD_CDict created from an empty dictionary.
+         But it is inefficient for small data, and it is recommended to use ZSTD_compressCCtx(). 
 


 
 
ZSTD_DDict* ZSTD_createDDict(const void* dictBuffer, size_t dictSize);
diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c
index e71e961f4..0f02540fc 100644
--- a/lib/decompress/zstd_decompress.c
+++ b/lib/decompress/zstd_decompress.c
@@ -1039,7 +1039,8 @@ size_t ZSTD_decodeSeqHeaders(ZSTD_DCtx* dctx, int* nbSeqPtr,
     /* prefetch dictionary content */
     if (dctx->ddictIsCold) {
         size_t const dictSize = (const char*)dctx->prefixStart - (const char*)dctx->virtualStart;
-        size_t const pSize = MIN(dictSize, (size_t)(64*nbSeq));
+        size_t const psmin = MIN(dictSize, (size_t)(64*nbSeq) /* heuristic */ );
+        size_t const pSize = MIN(psmin, 128 KB /* protection */ );
         const void* const pStart = (const char*)dctx->dictEnd - pSize;
         PREFETCH_AREA(pStart, pSize);
         dctx->ddictIsCold = 0;

From d195eec97e124eff513e4a254c1faa47ba450c9f Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Thu, 13 Sep 2018 12:29:52 -0700
Subject: [PATCH 264/372] fixed msan error

cold dictionary is detected through a comparison with dictEnd,
which was not initialized at the beginning of first DCtx usage.
---
 lib/decompress/zstd_decompress.c |  1 +
 tests/zstreamtest.c              | 28 ++++++++++++++--------------
 2 files changed, 15 insertions(+), 14 deletions(-)

diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c
index 0f02540fc..9d7408336 100644
--- a/lib/decompress/zstd_decompress.c
+++ b/lib/decompress/zstd_decompress.c
@@ -206,6 +206,7 @@ static void ZSTD_initDCtx_internal(ZSTD_DCtx* dctx)
     dctx->maxWindowSize = ZSTD_MAXWINDOWSIZE_DEFAULT;
     dctx->ddict       = NULL;
     dctx->ddictLocal  = NULL;
+    dctx->dictEnd     = NULL;
     dctx->ddictIsCold = 0;
     dctx->inBuff      = NULL;
     dctx->inBuffSize  = 0;
diff --git a/tests/zstreamtest.c b/tests/zstreamtest.c
index 0e0fbe0c9..96136a625 100644
--- a/tests/zstreamtest.c
+++ b/tests/zstreamtest.c
@@ -135,34 +135,34 @@ typedef struct {
     size_t filled;
 } buffer_t;
 
-static const buffer_t g_nullBuffer = { NULL, 0 , 0 };
+static const buffer_t kBuffNull = { NULL, 0 , 0 };
+
+static void FUZ_freeDictionary(buffer_t dict)
+{
+    free(dict.start);
+}
 
 static buffer_t FUZ_createDictionary(const void* src, size_t srcSize, size_t blockSize, size_t requestedDictSize)
 {
-    buffer_t dict = { NULL, 0, 0 };
+    buffer_t dict = kBuffNull;
     size_t const nbBlocks = (srcSize + (blockSize-1)) / blockSize;
-    size_t* const blockSizes = (size_t*) malloc(nbBlocks * sizeof(size_t));
-    if (!blockSizes) return dict;
+    size_t* const blockSizes = (size_t*)malloc(nbBlocks * sizeof(size_t));
+    if (!blockSizes) return kBuffNull;
     dict.start = malloc(requestedDictSize);
-    if (!dict.start) { free(blockSizes); return dict; }
+    if (!dict.start) { free(blockSizes); return kBuffNull; }
     {   size_t nb;
         for (nb=0; nb
Date: Thu, 13 Sep 2018 16:44:04 -0700
Subject: [PATCH 265/372] updated code comments, based on @terrelln review

---
 lib/common/compiler.h            | 8 +++++++-
 lib/decompress/zstd_decompress.c | 3 ++-
 2 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/lib/common/compiler.h b/lib/common/compiler.h
index 31eb1ccdf..07f875e4d 100644
--- a/lib/common/compiler.h
+++ b/lib/common/compiler.h
@@ -89,7 +89,13 @@
 #endif
 
 /* prefetch
- * can be disabled, by declaring NO_PREFETCH macro */
+ * can be disabled, by declaring NO_PREFETCH macro
+ * All prefetch invocations use a single default locality 2,
+ * generating instruction prefetcht1,
+ * which, according to Intel, means "load data into L2 cache".
+ * This is a good enough "middle ground" for the time being,
+ * though in theory, it would be better to specialize locality depending on data being prefetched.
+ * Tests could not determine any sensible difference based on locality value. */
 #if defined(NO_PREFETCH)
 #  define PREFETCH(ptr)     (void)(ptr)  /* disabled */
 #else
diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c
index 9d7408336..1382c9c7f 100644
--- a/lib/decompress/zstd_decompress.c
+++ b/lib/decompress/zstd_decompress.c
@@ -56,7 +56,8 @@
 *  Dependencies
 *********************************************************/
 #include       /* memcpy, memmove, memset */
-#include "cpu.h"         /* prefetch */
+#include "compiler.h"    /* prefetch */
+#include "cpu.h"         /* bmi2 */
 #include "mem.h"         /* low level memory routines */
 #define FSE_STATIC_LINKING_ONLY
 #include "fse.h"

From b048af5999061a41e68d1083ef7b0ab2fe75e29d Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" 
Date: Fri, 14 Sep 2018 15:23:35 -0700
Subject: [PATCH 266/372] ZSTD_fast: Don't Search Dict Context When Mismatch
 Was Found

---
 lib/compress/zstd_fast.c | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/lib/compress/zstd_fast.c b/lib/compress/zstd_fast.c
index 37a715167..7d7efb465 100644
--- a/lib/compress/zstd_fast.c
+++ b/lib/compress/zstd_fast.c
@@ -124,8 +124,7 @@ size_t ZSTD_compressBlock_fast_generic(
             mLength = ZSTD_count(ip+1+4, ip+1+4-offset_1, iend) + 4;
             ip++;
             ZSTD_storeSeq(seqStore, ip-anchor, anchor, 0, mLength-MINMATCH);
-        } else if ( (matchIndex <= prefixStartIndex)
-                 || (MEM_read32(match) != MEM_read32(ip)) ) {
+        } else if ( (matchIndex <= prefixStartIndex) ) {
             if (dictMode == ZSTD_dictMatchState) {
                 U32 const dictMatchIndex = dictHashTable[h];
                 const BYTE* dictMatch = dictBase + dictMatchIndex;
@@ -151,6 +150,11 @@ size_t ZSTD_compressBlock_fast_generic(
                 ip += ((ip-anchor) >> kSearchStrength) + stepSize;
                 continue;
             }
+        } else if (MEM_read32(match) != MEM_read32(ip)) {
+            /* it's not a match, and we're not going to check the dictionary */
+            assert(stepSize >= 1);
+            ip += ((ip-anchor) >> kSearchStrength) + stepSize;
+            continue;
         } else {
             /* found a regular match */
             U32 const offset = (U32)(ip-match);

From b76c88849748935bfd703a0703770d820e75c6b7 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" 
Date: Fri, 14 Sep 2018 15:24:25 -0700
Subject: [PATCH 267/372] ZSTD_dfast: Don't Search Dict Context When Mismatch
 Was Found

---
 lib/compress/zstd_double_fast.c | 54 ++++++++++++++++-----------------
 1 file changed, 27 insertions(+), 27 deletions(-)

diff --git a/lib/compress/zstd_double_fast.c b/lib/compress/zstd_double_fast.c
index 7fc11eb48..9231c8157 100644
--- a/lib/compress/zstd_double_fast.c
+++ b/lib/compress/zstd_double_fast.c
@@ -141,16 +141,16 @@ size_t ZSTD_compressBlock_doubleFast_generic(
             goto _match_stored;
         }
 
-        /* check prefix long match */
-        if ( (matchIndexL > prefixLowestIndex) && (MEM_read64(matchLong) == MEM_read64(ip)) ) {
-            mLength = ZSTD_count(ip+8, matchLong+8, iend) + 8;
-            offset = (U32)(ip-matchLong);
-            while (((ip>anchor) & (matchLong>prefixLowest)) && (ip[-1] == matchLong[-1])) { ip--; matchLong--; mLength++; } /* catch up */
-            goto _match_found;
-        }
-
-        /* check dictMatchState long match */
-        if (dictMode == ZSTD_dictMatchState) {
+        if (matchIndexL > prefixLowestIndex) {
+            /* check prefix long match */
+            if (MEM_read64(matchLong) == MEM_read64(ip)) {
+                mLength = ZSTD_count(ip+8, matchLong+8, iend) + 8;
+                offset = (U32)(ip-matchLong);
+                while (((ip>anchor) & (matchLong>prefixLowest)) && (ip[-1] == matchLong[-1])) { ip--; matchLong--; mLength++; } /* catch up */
+                goto _match_found;
+            }
+        } else if (dictMode == ZSTD_dictMatchState) {
+            /* check dictMatchState long match */
             U32 const dictMatchIndexL = dictHashLong[h2];
             const BYTE* dictMatchL = dictBase + dictMatchIndexL;
             assert(dictMatchL < dictEnd);
@@ -163,13 +163,13 @@ size_t ZSTD_compressBlock_doubleFast_generic(
             }
         }
 
-        /* check prefix short match */
-        if ( (matchIndexS > prefixLowestIndex) && (MEM_read32(match) == MEM_read32(ip)) ) {
-            goto _search_next_long;
-        }
-
-        /* check dictMatchState short match */
-        if (dictMode == ZSTD_dictMatchState) {
+        if (matchIndexS > prefixLowestIndex) {
+            /* check prefix short match */
+            if (MEM_read32(match) == MEM_read32(ip)) {
+                goto _search_next_long;
+            }
+        } else if (dictMode == ZSTD_dictMatchState) {
+            /* check dictMatchState short match */
             U32 const dictMatchIndexS = dictHashSmall[h];
             match = dictBase + dictMatchIndexS;
             matchIndexS = dictMatchIndexS + dictIndexDelta;
@@ -191,16 +191,16 @@ _search_next_long:
             hashLong[hl3] = current + 1;
 
             /* check prefix long +1 match */
-            if ( (matchIndexL3 > prefixLowestIndex) && (MEM_read64(matchL3) == MEM_read64(ip+1)) ) {
-                mLength = ZSTD_count(ip+9, matchL3+8, iend) + 8;
-                ip++;
-                offset = (U32)(ip-matchL3);
-                while (((ip>anchor) & (matchL3>prefixLowest)) && (ip[-1] == matchL3[-1])) { ip--; matchL3--; mLength++; } /* catch up */
-                goto _match_found;
-            }
-
-            /* check dict long +1 match */
-            if (dictMode == ZSTD_dictMatchState) {
+            if (matchIndexL3 > prefixLowestIndex) {
+                if (MEM_read64(matchL3) == MEM_read64(ip+1)) {
+                    mLength = ZSTD_count(ip+9, matchL3+8, iend) + 8;
+                    ip++;
+                    offset = (U32)(ip-matchL3);
+                    while (((ip>anchor) & (matchL3>prefixLowest)) && (ip[-1] == matchL3[-1])) { ip--; matchL3--; mLength++; } /* catch up */
+                    goto _match_found;
+                }
+            } else if (dictMode == ZSTD_dictMatchState) {
+                /* check dict long +1 match */
                 U32 const dictMatchIndexL3 = dictHashLong[hl3];
                 const BYTE* dictMatchL3 = dictBase + dictMatchIndexL3;
                 assert(dictMatchL3 < dictEnd);

From 7269fe6cd356d4006fe2dd9581aecf265d8f56c9 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Fri, 14 Sep 2018 16:06:35 -0700
Subject: [PATCH 268/372] minor code comment update

---
 doc/zstd_manual.html | 5 ++---
 lib/zstd.h           | 5 ++---
 2 files changed, 4 insertions(+), 6 deletions(-)

diff --git a/doc/zstd_manual.html b/doc/zstd_manual.html
index 5b9f07960..cdfeb7094 100644
--- a/doc/zstd_manual.html
+++ b/doc/zstd_manual.html
@@ -43,9 +43,8 @@
   The library supports regular compression levels from 1 up to ZSTD_maxCLevel(),
   which is currently 22. Levels >= 20, labeled `--ultra`, should be used with
   caution, as they require more memory. The library also offers negative
-  compression levels (all negative integers are valid levels), which extend the
-  range of speed vs. ratio preferences to increasingly extremely strongly
-  prioritize speed.
+  compression levels, which extend the range of speed vs. ratio preferences.
+  The lower the level, the faster the speed (at the cost of compression).
 
   Compression can be done in:
     - a single step (described as Simple API)
diff --git a/lib/zstd.h b/lib/zstd.h
index 72fc3590b..ac6f918a7 100644
--- a/lib/zstd.h
+++ b/lib/zstd.h
@@ -46,9 +46,8 @@ extern "C" {
   The library supports regular compression levels from 1 up to ZSTD_maxCLevel(),
   which is currently 22. Levels >= 20, labeled `--ultra`, should be used with
   caution, as they require more memory. The library also offers negative
-  compression levels (all negative integers are valid levels), which extend the
-  range of speed vs. ratio preferences to increasingly extremely strongly
-  prioritize speed.
+  compression levels, which extend the range of speed vs. ratio preferences.
+  The lower the level, the faster the speed (at the cost of compression).
 
   Compression can be done in:
     - a single step (described as Simple API)

From 18b4a1da618dd40fc9384120ff8205e9d239b896 Mon Sep 17 00:00:00 2001
From: ko-zu 
Date: Sun, 16 Sep 2018 10:27:02 +0900
Subject: [PATCH 269/372] Fix clang build

Fix dixygen comment
Fix clang binary path
---
 Makefile            | 2 +-
 lib/compress/hist.h | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/Makefile b/Makefile
index e29f0f749..a17900450 100644
--- a/Makefile
+++ b/Makefile
@@ -215,7 +215,7 @@ gcc6test: clean
 
 clangtest: clean
 	clang -v
-	$(MAKE) all CXX=clang-++ CC=clang MOREFLAGS="-Werror -Wconversion -Wno-sign-conversion -Wdocumentation"
+	$(MAKE) all CXX=clang++ CC=clang MOREFLAGS="-Werror -Wconversion -Wno-sign-conversion -Wdocumentation"
 
 armtest: clean
 	$(MAKE) -C $(TESTDIR) datagen   # use native, faster
diff --git a/lib/compress/hist.h b/lib/compress/hist.h
index 788470da7..8b1991a90 100644
--- a/lib/compress/hist.h
+++ b/lib/compress/hist.h
@@ -50,7 +50,7 @@
 size_t HIST_count(unsigned* count, unsigned* maxSymbolValuePtr,
                   const void* src, size_t srcSize);
 
-unsigned HIST_isError(size_t code);  /*< tells if a return value is an error code */
+unsigned HIST_isError(size_t code);  /**< tells if a return value is an error code */
 
 
 /* --- advanced histogram functions --- */

From b52867a97f712ec2d7a666bbb592f4fa6c8e6907 Mon Sep 17 00:00:00 2001
From: Azat Khuzhin 
Date: Sun, 16 Sep 2018 18:04:43 +0300
Subject: [PATCH 270/372] zstdseek_decompress: fix decompression with data left
 in input buffer

---
 contrib/seekable_format/zstdseek_decompress.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/contrib/seekable_format/zstdseek_decompress.c b/contrib/seekable_format/zstdseek_decompress.c
index 2b109b9d0..b4c48754e 100644
--- a/contrib/seekable_format/zstdseek_decompress.c
+++ b/contrib/seekable_format/zstdseek_decompress.c
@@ -313,8 +313,8 @@ static size_t ZSTD_seekable_loadSeekTable(ZSTD_seekable* zs)
             /* compute cumulative positions */
             for (; idx < numFrames; idx++) {
                 if (pos + sizePerEntry > SEEKABLE_BUFF_SIZE) {
-                    U32 const toRead = MIN(remaining, SEEKABLE_BUFF_SIZE);
                     U32 const offset = SEEKABLE_BUFF_SIZE - pos;
+                    U32 const toRead = MIN(remaining, SEEKABLE_BUFF_SIZE - offset);
                     memmove(zs->inBuff, zs->inBuff + pos, offset); /* move any data we haven't read yet */
                     CHECK_IO(src.read(src.opaque, zs->inBuff+offset, toRead));
                     remaining -= toRead;

From d707692e05df1faec8e7437f08cb40623223ed1b Mon Sep 17 00:00:00 2001
From: Azat Khuzhin 
Date: Sun, 16 Sep 2018 18:04:43 +0300
Subject: [PATCH 271/372] seekable_decompression: support offset greater then
 UNIT_MAX

---
 contrib/seekable_format/examples/seekable_decompression.c | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/contrib/seekable_format/examples/seekable_decompression.c b/contrib/seekable_format/examples/seekable_decompression.c
index 9cd232922..7050e0fa5 100644
--- a/contrib/seekable_format/examples/seekable_decompression.c
+++ b/contrib/seekable_format/examples/seekable_decompression.c
@@ -84,7 +84,7 @@ static void fseek_orDie(FILE* file, long int offset, int origin) {
 }
 
 
-static void decompressFile_orDie(const char* fname, unsigned startOffset, unsigned endOffset)
+static void decompressFile_orDie(const char* fname, off_t startOffset, off_t endOffset)
 {
     FILE* const fin  = fopen_orDie(fname, "rb");
     FILE* const fout = stdout;
@@ -129,8 +129,8 @@ int main(int argc, const char** argv)
 
     {
         const char* const inFilename = argv[1];
-        unsigned const startOffset = (unsigned) atoi(argv[2]);
-        unsigned const endOffset = (unsigned) atoi(argv[3]);
+        off_t const startOffset = atoll(argv[2]);
+        off_t const endOffset = atoll(argv[3]);
         decompressFile_orDie(inFilename, startOffset, endOffset);
     }
 

From b053bec2f495a46c925f6ad79780bbc61a110a40 Mon Sep 17 00:00:00 2001
From: ko-zu 
Date: Mon, 17 Sep 2018 13:09:08 +0900
Subject: [PATCH 272/372] Fix largeNbDicts bench for clangbuild

Remove unsigned to size_t promotion to fix implicit down conversion errors in clangbuild target.
---
 contrib/largeNbDicts/largeNbDicts.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/contrib/largeNbDicts/largeNbDicts.c b/contrib/largeNbDicts/largeNbDicts.c
index db9e730b3..e0bc55380 100644
--- a/contrib/largeNbDicts/largeNbDicts.c
+++ b/contrib/largeNbDicts/largeNbDicts.c
@@ -766,8 +766,8 @@ int main (int argc, const char** argv)
     const char* dictionary = NULL;
     int cLevel = CLEVEL_DEFAULT;
     size_t blockSize = BLOCKSIZE_DEFAULT;
-    size_t nbDicts = 0;  /* determine nbDicts automatically: 1 dictionary per block */
-    size_t nbBlocks = 0; /* determine nbBlocks automatically, from source and blockSize */
+    unsigned nbDicts = 0;  /* determine nbDicts automatically: 1 dictionary per block */
+    unsigned nbBlocks = 0; /* determine nbBlocks automatically, from source and blockSize */
 
     for (int argNb = 1; argNb < argc ; argNb++) {
         const char* argument = argv[argNb];

From 06fd1e473d2435b3a58741a8cfca802fbf4aa5a2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Ketelaars?= 
Date: Sat, 30 Jun 2018 15:42:42 +0200
Subject: [PATCH 273/372] 'head -c BYTES' is non-portable.

tests/playTests.sh uses 'head -c' in a couple of tests to truncate the
last byte of a file. The '-c' option is non-portable (not in POSIX).
Instead use a wrapper around dd (truncateLastByte).
---
 tests/playTests.sh | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/tests/playTests.sh b/tests/playTests.sh
index fa0685c6d..19b502999 100755
--- a/tests/playTests.sh
+++ b/tests/playTests.sh
@@ -48,6 +48,10 @@ fileRoundTripTest() {
     $DIFF -q tmp.md5.1 tmp.md5.2
 }
 
+truncateLastByte() {
+	dd bs=1 count=$(($(wc -c < "$1") - 1)) if="$1" status=none
+}
+
 UNAME=$(uname)
 
 isTerminal=false
@@ -592,7 +596,7 @@ if [ $GZIPMODE -eq 1 ]; then
     $ZSTD -f --format=gzip tmp
     $ZSTD -f tmp
     cat tmp.gz tmp.zst tmp.gz tmp.zst | $ZSTD -d -f -o tmp
-    head -c -1 tmp.gz | $ZSTD -t > $INTOVOID && die "incomplete frame not detected !"
+    truncateLastByte tmp.gz | $ZSTD -t > $INTOVOID && die "incomplete frame not detected !"
     rm tmp*
 else
     $ECHO "gzip mode not supported"
@@ -659,8 +663,8 @@ if [ $LZMAMODE -eq 1 ]; then
     $ZSTD -f --format=lzma tmp
     $ZSTD -f tmp
     cat tmp.xz tmp.lzma tmp.zst tmp.lzma tmp.xz tmp.zst | $ZSTD -d -f -o tmp
-    head -c -1 tmp.xz | $ZSTD -t > $INTOVOID && die "incomplete frame not detected !"
-    head -c -1 tmp.lzma | $ZSTD -t > $INTOVOID && die "incomplete frame not detected !"
+    truncateLastByte tmp.xz | $ZSTD -t > $INTOVOID && die "incomplete frame not detected !"
+    truncateLastByte tmp.lzma | $ZSTD -t > $INTOVOID && die "incomplete frame not detected !"
     rm tmp*
 else
     $ECHO "xz mode not supported"
@@ -696,7 +700,7 @@ if [ $LZ4MODE -eq 1 ]; then
     $ZSTD -f --format=lz4 tmp
     $ZSTD -f tmp
     cat tmp.lz4 tmp.zst tmp.lz4 tmp.zst | $ZSTD -d -f -o tmp
-    head -c -1 tmp.lz4 | $ZSTD -t > $INTOVOID && die "incomplete frame not detected !"
+    truncateLastByte tmp.lz4 | $ZSTD -t > $INTOVOID && die "incomplete frame not detected !"
     rm tmp*
 else
     $ECHO "lz4 mode not supported"

From 005f000aed01293bc301dab05e16c7b45a689d3e Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Tue, 18 Sep 2018 13:07:08 -0700
Subject: [PATCH 274/372] updated documentation of *refPrefix()

indicating the equivalence with `diff` operation.
---
 doc/zstd_manual.html | 15 +++++++++++----
 lib/zstd.h           | 15 +++++++++++----
 2 files changed, 22 insertions(+), 8 deletions(-)

diff --git a/doc/zstd_manual.html b/doc/zstd_manual.html
index cdfeb7094..5e01a19c3 100644
--- a/doc/zstd_manual.html
+++ b/doc/zstd_manual.html
@@ -979,16 +979,21 @@ size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx* cctx,
                            const void* prefix, size_t prefixSize,
                            ZSTD_dictContentType_e dictContentType);
 
  Reference a prefix (single-usage dictionary) for next compression job.
-  Decompression need same prefix to properly regenerate data.
-  Prefix is **only used once**. Tables are discarded at end of compression job (ZSTD_e_end).
+  Decompression will need same prefix to properly regenerate data.
+  Compressing with a prefix is similar in outcome as performing a diff and compressing it,
+  but performs much faster, especially during decompression (compression speed is tunable with compression level).
+  Note that prefix is **only used once**. Tables are discarded at end of compression job (ZSTD_e_end).
  @result : 0, or an error code (which can be tested with ZSTD_isError()).
   Special: Adding any prefix (including NULL) invalidates any previous prefix or dictionary
   Note 1 : Prefix buffer is referenced. It **must** outlive compression job.
            Its contain must remain unmodified up to end of compression (ZSTD_e_end).
-  Note 2 : Referencing a prefix involves building tables, which are dependent on compression parameters.
+  Note 2 : If the intention is to diff some large src data blob with some prior version of itself,
+           ensure that the window size is large enough to contain the entire source.
+           See ZSTD_p_windowLog.
+  Note 3 : Referencing a prefix involves building tables, which are dependent on compression parameters.
            It's a CPU consuming operation, with non-negligible impact on latency.
            If there is a need to use same prefix multiple times, consider loadDictionary instead.
-  Note 3 : By default, the prefix is treated as raw content (ZSTD_dm_rawContent).
+  Note 4 : By default, the prefix is treated as raw content (ZSTD_dm_rawContent).
            Use ZSTD_CCtx_refPrefix_advanced() to alter dictMode. 
 


 
@@ -1155,6 +1160,8 @@ size_t ZSTD_DCtx_refPrefix_advanced(ZSTD_DCtx* dctx,
                         const void* prefix, size_t prefixSize,
                         ZSTD_dictContentType_e dictContentType);
 
  Reference a prefix (single-usage dictionary) for next compression job.
+  This is the reverse operation of ZSTD_CCtx_refPrefix(),
+  and must use the same prefix as the one used during compression.
   Prefix is **only used once**. Reference is discarded at end of frame.
   End of frame is reached when ZSTD_DCtx_decompress_generic() returns 0.
  @result : 0, or an error code (which can be tested with ZSTD_isError()).
diff --git a/lib/zstd.h b/lib/zstd.h
index ac6f918a7..277ab7ffb 100644
--- a/lib/zstd.h
+++ b/lib/zstd.h
@@ -1164,16 +1164,21 @@ ZSTDLIB_API size_t ZSTD_CCtx_refCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict);
 
 /*! ZSTD_CCtx_refPrefix() :
  *  Reference a prefix (single-usage dictionary) for next compression job.
- *  Decompression need same prefix to properly regenerate data.
- *  Prefix is **only used once**. Tables are discarded at end of compression job (ZSTD_e_end).
+ *  Decompression will need same prefix to properly regenerate data.
+ *  Compressing with a prefix is similar in outcome as performing a diff and compressing it,
+ *  but performs much faster, especially during decompression (compression speed is tunable with compression level).
+ *  Note that prefix is **only used once**. Tables are discarded at end of compression job (ZSTD_e_end).
  * @result : 0, or an error code (which can be tested with ZSTD_isError()).
  *  Special: Adding any prefix (including NULL) invalidates any previous prefix or dictionary
  *  Note 1 : Prefix buffer is referenced. It **must** outlive compression job.
  *           Its contain must remain unmodified up to end of compression (ZSTD_e_end).
- *  Note 2 : Referencing a prefix involves building tables, which are dependent on compression parameters.
+ *  Note 2 : If the intention is to diff some large src data blob with some prior version of itself,
+ *           ensure that the window size is large enough to contain the entire source.
+ *           See ZSTD_p_windowLog.
+ *  Note 3 : Referencing a prefix involves building tables, which are dependent on compression parameters.
  *           It's a CPU consuming operation, with non-negligible impact on latency.
  *           If there is a need to use same prefix multiple times, consider loadDictionary instead.
- *  Note 3 : By default, the prefix is treated as raw content (ZSTD_dm_rawContent).
+ *  Note 4 : By default, the prefix is treated as raw content (ZSTD_dm_rawContent).
  *           Use ZSTD_CCtx_refPrefix_advanced() to alter dictMode. */
 ZSTDLIB_API size_t ZSTD_CCtx_refPrefix(ZSTD_CCtx* cctx,
                                        const void* prefix, size_t prefixSize);
@@ -1356,6 +1361,8 @@ ZSTDLIB_API size_t ZSTD_DCtx_refDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict);
 
 /*! ZSTD_DCtx_refPrefix() :
  *  Reference a prefix (single-usage dictionary) for next compression job.
+ *  This is the reverse operation of ZSTD_CCtx_refPrefix(),
+ *  and must use the same prefix as the one used during compression.
  *  Prefix is **only used once**. Reference is discarded at end of frame.
  *  End of frame is reached when ZSTD_DCtx_decompress_generic() returns 0.
  * @result : 0, or an error code (which can be tested with ZSTD_isError()).

From 89bc309d904d1472288b9458687994987a8ce68d Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Wed, 19 Sep 2018 14:49:13 -0700
Subject: [PATCH 275/372] error out when --adapt is associated with
 --single-thread

since they are not compatible
---
 lib/zstd.h        | 27 +++++++++++++++++----------
 programs/fileio.c | 37 ++++++++++++++++++++++---------------
 2 files changed, 39 insertions(+), 25 deletions(-)

diff --git a/lib/zstd.h b/lib/zstd.h
index 25441e68c..669161b41 100644
--- a/lib/zstd.h
+++ b/lib/zstd.h
@@ -736,13 +736,14 @@ ZSTDLIB_API size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs, const
 
 /*! ZSTD_resetCStream() :
  *  start a new compression job, using same parameters from previous job.
- *  This is typically useful to skip dictionary loading stage, since it will re-use it in-place..
+ *  This is typically useful to skip dictionary loading stage, since it will re-use it in-place.
  *  Note that zcs must be init at least once before using ZSTD_resetCStream().
  *  If pledgedSrcSize is not known at reset time, use macro ZSTD_CONTENTSIZE_UNKNOWN.
  *  If pledgedSrcSize > 0, its value must be correct, as it will be written in header, and controlled at the end.
  *  For the time being, pledgedSrcSize==0 is interpreted as "srcSize unknown" for compatibility with older programs,
  *  but it will change to mean "empty" in future version, so use macro ZSTD_CONTENTSIZE_UNKNOWN instead.
- * @return : 0, or an error code (which can be tested using ZSTD_isError()) */
+ * @return : 0, or an error code (which can be tested using ZSTD_isError())
+ */
 ZSTDLIB_API size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize);
 
 
@@ -755,21 +756,27 @@ typedef struct {
     unsigned nbActiveWorkers;      /* MT only : nb of workers actively compressing at probe time */
 } ZSTD_frameProgression;
 
-/* ZSTD_getFrameProgression():
+/* ZSTD_getFrameProgression() :
  * tells how much data has been ingested (read from input)
  * consumed (input actually compressed) and produced (output) for current frame.
- * Therefore, (ingested - consumed) is amount of input data buffered internally, not yet compressed.
- * Can report progression inside worker threads (multi-threading and non-blocking mode).
+ * Note : (ingested - consumed) is amount of input data buffered internally, not yet compressed.
+ * Aggregates progression inside active worker threads.
  */
 ZSTDLIB_API ZSTD_frameProgression ZSTD_getFrameProgression(const ZSTD_CCtx* cctx);
 
-/*! ZSTD_toFlushNow()
+/*! ZSTD_toFlushNow() :
  *  Tell how many bytes are ready to be flushed immediately.
  *  Useful for multithreading scenarios (nbWorkers >= 1).
- *  Probe the oldest active job (not yet entirely flushed) and check its output buffer.
- *  If return 0, it means there is no active job, or
- *  it means oldest job is still active, but everything produced has been flushed so far,
- *  therefore flushing is limited by speed of oldest job. */
+ *  Probe the oldest active job, defined as oldest job not yet entirely flushed,
+ *  and check its output buffer.
+ * @return : amount of data stored in oldest job and ready to be flushed immediately.
+ *  if @return == 0, it means either :
+ *  + there is no active job (could be checked with ZSTD_frameProgression()), or
+ *  + oldest job is still actively compressing data,
+ *    but everything it has produced has also been flushed so far,
+ *    therefore flushing speed is currently limited by production speed of oldest job
+ *    irrespective of the speed of concurrent newer jobs.
+ */
 ZSTDLIB_API size_t ZSTD_toFlushNow(ZSTD_CCtx* cctx);
 
 
diff --git a/programs/fileio.c b/programs/fileio.c
index 6ea43c902..701e30e8f 100644
--- a/programs/fileio.c
+++ b/programs/fileio.c
@@ -283,7 +283,11 @@ void FIO_setOverlapLog(unsigned overlapLog){
     g_overlapLog = overlapLog;
 }
 static U32 g_adaptiveMode = 0;
-void FIO_setAdaptiveMode(unsigned adapt) { g_adaptiveMode = adapt; }
+void FIO_setAdaptiveMode(unsigned adapt) {
+    if ((adapt>0) && (g_nbWorkers==0))
+        EXM_THROW(1, "Adaptive mode is not compatible with single thread mode \n");
+    g_adaptiveMode = adapt;
+}
 static U32 g_ldmFlag = 0;
 void FIO_setLdmFlag(unsigned ldmFlag) {
     g_ldmFlag = (ldmFlag>0);
@@ -541,7 +545,8 @@ static void FIO_freeCResources(cRess_t ress)
 
 
 #ifdef ZSTD_GZCOMPRESS
-static unsigned long long FIO_compressGzFrame(cRess_t* ress,
+static unsigned long long
+FIO_compressGzFrame(cRess_t* ress,
                     const char* srcFileName, U64 const srcFileSize,
                     int compressionLevel, U64* readsize)
 {
@@ -623,9 +628,10 @@ static unsigned long long FIO_compressGzFrame(cRess_t* ress,
 
 
 #ifdef ZSTD_LZMACOMPRESS
-static unsigned long long FIO_compressLzmaFrame(cRess_t* ress,
-                            const char* srcFileName, U64 const srcFileSize,
-                            int compressionLevel, U64* readsize, int plain_lzma)
+static unsigned long long
+FIO_compressLzmaFrame(cRess_t* ress,
+                      const char* srcFileName, U64 const srcFileSize,
+                      int compressionLevel, U64* readsize, int plain_lzma)
 {
     unsigned long long inFileSize = 0, outFileSize = 0;
     lzma_stream strm = LZMA_STREAM_INIT;
@@ -698,9 +704,10 @@ static unsigned long long FIO_compressLzmaFrame(cRess_t* ress,
 #define LZ4F_max64KB max64KB
 #endif
 static int FIO_LZ4_GetBlockSize_FromBlockId (int id) { return (1 << (8 + (2 * id))); }
-static unsigned long long FIO_compressLz4Frame(cRess_t* ress,
-                            const char* srcFileName, U64 const srcFileSize,
-                            int compressionLevel, U64* readsize)
+static unsigned long long
+FIO_compressLz4Frame(cRess_t* ress,
+                     const char* srcFileName, U64 const srcFileSize,
+                     int compressionLevel, U64* readsize)
 {
     const size_t blockSize = FIO_LZ4_GetBlockSize_FromBlockId(LZ4F_max64KB);
     unsigned long long inFileSize = 0, outFileSize = 0;
@@ -838,7 +845,7 @@ FIO_compressZstdFrame(const cRess_t* ressPtr,
 
             /* count stats */
             inputPresented++;
-            if (oldIPos == inBuff.pos) inputBlocked++;
+            if (oldIPos == inBuff.pos) inputBlocked++;  /* input buffer is full and can't take any more : input speed is faster than consumption rate */
             if (!toFlushNow) flushWaiting = 1;
 
             /* Write compressed stream */
@@ -846,7 +853,7 @@ FIO_compressZstdFrame(const cRess_t* ressPtr,
                             (U32)directive, (U32)inBuff.pos, (U32)inBuff.size, (U32)outBuff.pos);
             if (outBuff.pos) {
                 size_t const sizeCheck = fwrite(ress.dstBuffer, 1, outBuff.pos, dstFile);
-                if (sizeCheck!=outBuff.pos)
+                if (sizeCheck != outBuff.pos)
                     EXM_THROW(25, "Write error : cannot write compressed block");
                 compressedfilesize += outBuff.pos;
             }
@@ -857,24 +864,24 @@ FIO_compressZstdFrame(const cRess_t* ressPtr,
                 double const cShare = (double)zfp.produced / (zfp.consumed + !zfp.consumed/*avoid div0*/) * 100;
 
                 /* check output speed */
-                if (zfp.currentJobID > 1) {
-                    static ZSTD_frameProgression cpszfp = { 0, 0, 0, 0, 0, 0 };
+                if (zfp.currentJobID > 1) {  /* only possible if nbWorkers >= 1 */
+                    static ZSTD_frameProgression cpszfp = { 0, 0, 0, 0, 0, 0 };   /* note : requires fileio to run main thread */
 
                     unsigned long long newlyProduced = zfp.produced - cpszfp.produced;
                     unsigned long long newlyFlushed = zfp.flushed - cpszfp.flushed;
                     assert(zfp.produced >= cpszfp.produced);
-
-                    cpszfp = zfp;
+                    assert(g_nbWorkers >= 1);
 
                     if ( (zfp.ingested == cpszfp.ingested)   /* no data read : input buffer full */
                       && (zfp.consumed == cpszfp.consumed)   /* no data compressed : no more buffer to compress OR compression is really slow */
                       && (zfp.nbActiveWorkers == 0)          /* confirmed : no compression : either no more buffer to compress OR not enough data to start first worker */
-                      && (zfp.currentJobID > 0)              /* first job started : only remaining reason is no more available buffer to start compression */
                       ) {
                         DISPLAYLEVEL(6, "all buffers full : compression stopped => slow down \n")
                         speedChange = slower;
                     }
 
+                    cpszfp = zfp;
+
                     if ( (newlyProduced > (newlyFlushed * 9 / 8))   /* compression produces more data than output can flush (though production can be spiky, due to work unit : (N==4)*block sizes) */
                       && (flushWaiting == 0)                        /* flush speed was never slowed by lack of production, so it's operating at max capacity */
                       ) {

From ca02ebee0795ba893db6368a20f09f02311698cc Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Wed, 19 Sep 2018 15:09:45 -0700
Subject: [PATCH 276/372] removed static variables

so that --adapt can work on multiple input files too
---
 lib/compress/zstdmt_compress.c |   2 +-
 programs/fileio.c              | 157 +++++++++++++++++----------------
 2 files changed, 81 insertions(+), 78 deletions(-)

diff --git a/lib/compress/zstdmt_compress.c b/lib/compress/zstdmt_compress.c
index 244690cdc..39255fdcf 100644
--- a/lib/compress/zstdmt_compress.c
+++ b/lib/compress/zstdmt_compress.c
@@ -1530,7 +1530,7 @@ static size_t ZSTDMT_createCompressionJob(ZSTDMT_CCtx* mtctx, size_t srcSize, ZS
         mtctx->jobs[jobID].jobID = mtctx->nextJobID;
         mtctx->jobs[jobID].firstJob = (mtctx->nextJobID==0);
         mtctx->jobs[jobID].lastJob = endFrame;
-        mtctx->jobs[jobID].frameChecksumNeeded = endFrame && (mtctx->nextJobID>0) && mtctx->params.fParams.checksumFlag;
+        mtctx->jobs[jobID].frameChecksumNeeded = mtctx->params.fParams.checksumFlag && endFrame && (mtctx->nextJobID>0);
         mtctx->jobs[jobID].dstFlushed = 0;
 
         /* Update the round buffer pos and clear the input buffer to be reset */
diff --git a/programs/fileio.c b/programs/fileio.c
index 701e30e8f..00f0bc263 100644
--- a/programs/fileio.c
+++ b/programs/fileio.c
@@ -807,6 +807,8 @@ FIO_compressZstdFrame(const cRess_t* ressPtr,
     ZSTD_EndDirective directive = ZSTD_e_continue;
 
     /* stats */
+    ZSTD_frameProgression previous_zfp_update = { 0, 0, 0, 0, 0, 0 };
+    ZSTD_frameProgression previous_zfp_correction = { 0, 0, 0, 0, 0, 0 };
     typedef enum { noChange, slower, faster } speedChange_e;
     speedChange_e speedChange = noChange;
     unsigned flushWaiting = 0;
@@ -820,7 +822,7 @@ FIO_compressZstdFrame(const cRess_t* ressPtr,
     if (fileSize != UTIL_FILESIZE_UNKNOWN) {
         CHECK(ZSTD_CCtx_setPledgedSrcSize(ress.cctx, fileSize));
     }
-    (void)compressionLevel; (void)srcFileName;
+    (void)srcFileName;
 
     /* Main compression loop */
     do {
@@ -863,69 +865,85 @@ FIO_compressZstdFrame(const cRess_t* ressPtr,
                 ZSTD_frameProgression const zfp = ZSTD_getFrameProgression(ress.cctx);
                 double const cShare = (double)zfp.produced / (zfp.consumed + !zfp.consumed/*avoid div0*/) * 100;
 
-                /* check output speed */
-                if (zfp.currentJobID > 1) {  /* only possible if nbWorkers >= 1 */
-                    static ZSTD_frameProgression cpszfp = { 0, 0, 0, 0, 0, 0 };   /* note : requires fileio to run main thread */
-
-                    unsigned long long newlyProduced = zfp.produced - cpszfp.produced;
-                    unsigned long long newlyFlushed = zfp.flushed - cpszfp.flushed;
-                    assert(zfp.produced >= cpszfp.produced);
-                    assert(g_nbWorkers >= 1);
-
-                    if ( (zfp.ingested == cpszfp.ingested)   /* no data read : input buffer full */
-                      && (zfp.consumed == cpszfp.consumed)   /* no data compressed : no more buffer to compress OR compression is really slow */
-                      && (zfp.nbActiveWorkers == 0)          /* confirmed : no compression : either no more buffer to compress OR not enough data to start first worker */
-                      ) {
-                        DISPLAYLEVEL(6, "all buffers full : compression stopped => slow down \n")
-                        speedChange = slower;
-                    }
-
-                    cpszfp = zfp;
-
-                    if ( (newlyProduced > (newlyFlushed * 9 / 8))   /* compression produces more data than output can flush (though production can be spiky, due to work unit : (N==4)*block sizes) */
-                      && (flushWaiting == 0)                        /* flush speed was never slowed by lack of production, so it's operating at max capacity */
-                      ) {
-                        DISPLAYLEVEL(6, "compression faster than flush (%llu > %llu), and flushed was never slowed down by lack of production => slow down \n", newlyProduced, newlyFlushed);
-                        speedChange = slower;
-                    }
-                    flushWaiting = 0;
+                /* display progress notifications */
+                if (g_displayLevel >= 3) {
+                    DISPLAYUPDATE(3, "\r(L%i) Buffered :%4u MB - Consumed :%4u MB - Compressed :%4u MB => %.2f%% ",
+                                compressionLevel,
+                                (U32)((zfp.ingested - zfp.consumed) >> 20),
+                                (U32)(zfp.consumed >> 20),
+                                (U32)(zfp.produced >> 20),
+                                cShare );
+                } else {   /* summarized notifications if == 2; */
+                    DISPLAYLEVEL(2, "\rRead : %u ", (U32)(zfp.consumed >> 20));
+                    if (fileSize != UTIL_FILESIZE_UNKNOWN)
+                        DISPLAYLEVEL(2, "/ %u ", (U32)(fileSize >> 20));
+                    DISPLAYLEVEL(2, "MB ==> %2.f%% ", cShare);
+                    DELAY_NEXT_UPDATE();
                 }
 
-                /* course correct only if there is at least one new job completed */
-                if (zfp.currentJobID > lastJobID) {
-                    DISPLAYLEVEL(6, "compression level adaptation check \n")
+                /* adaptive mode : statistics measurement and speed correction */
+                if (g_adaptiveMode) {
 
-                    /* check input speed */
-                    if (zfp.currentJobID > g_nbWorkers+1) {   /* warm up period, to fill all workers */
-                        if (inputBlocked <= 0) {
-                            DISPLAYLEVEL(6, "input is never blocked => input is too slow \n");
+                    /* check output speed */
+                    if (zfp.currentJobID > 1) {  /* only possible if nbWorkers >= 1 */
+
+                        unsigned long long newlyProduced = zfp.produced - previous_zfp_update.produced;
+                        unsigned long long newlyFlushed = zfp.flushed - previous_zfp_update.flushed;
+                        assert(zfp.produced >= previous_zfp_update.produced);
+                        assert(g_nbWorkers >= 1);
+
+                        if ( (zfp.ingested == previous_zfp_update.ingested)   /* no data read : input buffer full */
+                          && (zfp.consumed == previous_zfp_update.consumed)   /* no data compressed : no more buffer to compress OR compression is really slow */
+                          && (zfp.nbActiveWorkers == 0)          /* confirmed : no compression : either no more buffer to compress OR not enough data to start first worker */
+                          ) {
+                            DISPLAYLEVEL(6, "all buffers full : compression stopped => slow down \n")
                             speedChange = slower;
-                        } else if (speedChange == noChange) {
-                            static ZSTD_frameProgression csuzfp = { 0, 0, 0, 0, 0, 0 };
-                            unsigned long long newlyIngested = zfp.ingested - csuzfp.ingested;
-                            unsigned long long newlyConsumed = zfp.consumed - csuzfp.consumed;
-                            unsigned long long newlyProduced = zfp.produced - csuzfp.produced;
-                            unsigned long long newlyFlushed = zfp.flushed - csuzfp.flushed;
-                            csuzfp = zfp;
-                            assert(inputPresented > 0);
-                            DISPLAYLEVEL(6, "input blocked %u/%u(%.2f) - ingested:%u vs %u:consumed - flushed:%u vs %u:produced \n",
-                                            inputBlocked, inputPresented, (double)inputBlocked/inputPresented*100,
-                                            (U32)newlyIngested, (U32)newlyConsumed,
-                                            (U32)newlyFlushed, (U32)newlyProduced);
-                            if ( (inputBlocked > inputPresented / 8)     /* input is waiting often, because input buffers is full : compression or output too slow */
-                              && (newlyFlushed * 33 / 32 > newlyProduced)  /* flush everything that is produced */
-                              && (newlyIngested * 33 / 32 > newlyConsumed) /* input speed as fast or faster than compression speed */
-                            ) {
-                                DISPLAYLEVEL(6, "recommend faster as in(%llu) >= (%llu)comp(%llu) <= out(%llu) \n",
-                                                newlyIngested, newlyConsumed, newlyProduced, newlyFlushed);
-                                speedChange = faster;
-                            }
                         }
-                        inputBlocked = 0;
-                        inputPresented = 0;
+
+                        previous_zfp_update = zfp;
+
+                        if ( (newlyProduced > (newlyFlushed * 9 / 8))   /* compression produces more data than output can flush (though production can be spiky, due to work unit : (N==4)*block sizes) */
+                          && (flushWaiting == 0)                        /* flush speed was never slowed by lack of production, so it's operating at max capacity */
+                          ) {
+                            DISPLAYLEVEL(6, "compression faster than flush (%llu > %llu), and flushed was never slowed down by lack of production => slow down \n", newlyProduced, newlyFlushed);
+                            speedChange = slower;
+                        }
+                        flushWaiting = 0;
                     }
 
-                    if (g_adaptiveMode) {
+                    /* course correct only if there is at least one new job completed */
+                    if (zfp.currentJobID > lastJobID) {
+                        DISPLAYLEVEL(6, "compression level adaptation check \n")
+
+                        /* check input speed */
+                        if (zfp.currentJobID > g_nbWorkers+1) {   /* warm up period, to fill all workers */
+                            if (inputBlocked <= 0) {
+                                DISPLAYLEVEL(6, "input is never blocked => input is slower than ingestion \n");
+                                speedChange = slower;
+                            } else if (speedChange == noChange) {
+                                unsigned long long newlyIngested = zfp.ingested - previous_zfp_correction.ingested;
+                                unsigned long long newlyConsumed = zfp.consumed - previous_zfp_correction.consumed;
+                                unsigned long long newlyProduced = zfp.produced - previous_zfp_correction.produced;
+                                unsigned long long newlyFlushed  = zfp.flushed  - previous_zfp_correction.flushed;
+                                previous_zfp_correction = zfp;
+                                assert(inputPresented > 0);
+                                DISPLAYLEVEL(6, "input blocked %u/%u(%.2f) - ingested:%u vs %u:consumed - flushed:%u vs %u:produced \n",
+                                                inputBlocked, inputPresented, (double)inputBlocked/inputPresented*100,
+                                                (U32)newlyIngested, (U32)newlyConsumed,
+                                                (U32)newlyFlushed, (U32)newlyProduced);
+                                if ( (inputBlocked > inputPresented / 8)     /* input is waiting often, because input buffers is full : compression or output too slow */
+                                  && (newlyFlushed * 33 / 32 > newlyProduced)  /* flush everything that is produced */
+                                  && (newlyIngested * 33 / 32 > newlyConsumed) /* input speed as fast or faster than compression speed */
+                                ) {
+                                    DISPLAYLEVEL(6, "recommend faster as in(%llu) >= (%llu)comp(%llu) <= out(%llu) \n",
+                                                    newlyIngested, newlyConsumed, newlyProduced, newlyFlushed);
+                                    speedChange = faster;
+                                }
+                            }
+                            inputBlocked = 0;
+                            inputPresented = 0;
+                        }
+
                         if (speedChange == slower) {
                             DISPLAYLEVEL(6, "slower speed , higher compression \n")
                             compressionLevel ++;
@@ -940,27 +958,12 @@ FIO_compressZstdFrame(const cRess_t* ressPtr,
                             ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_compressionLevel, (unsigned)compressionLevel);
                         }
                         speedChange = noChange;
-                    }
-                    lastJobID = zfp.currentJobID;
-                }    /* if (zfp.currentJobID > lastJobID) */
 
-                if (g_displayLevel >= 3) {
-                    DISPLAYUPDATE(3, "\r(L%i) Buffered :%4u MB - Consumed :%4u MB - Compressed :%4u MB => %.2f%% ",
-                                compressionLevel,
-                                (U32)((zfp.ingested - zfp.consumed) >> 20),
-                                (U32)(zfp.consumed >> 20),
-                                (U32)(zfp.produced >> 20),
-                                cShare );
-                } else {
-                    /* g_displayLevel <= 2; only display notifications if == 2; */
-                    DISPLAYLEVEL(2, "\rRead : %u ", (U32)(zfp.consumed >> 20));
-                    if (fileSize != UTIL_FILESIZE_UNKNOWN)
-                        DISPLAYLEVEL(2, "/ %u ", (U32)(fileSize >> 20));
-                    DISPLAYLEVEL(2, "MB ==> %2.f%% ", cShare);
-                    DELAY_NEXT_UPDATE();
-                }
-            }
-        }
+                        lastJobID = zfp.currentJobID;
+                    }  /* if (zfp.currentJobID > lastJobID) */
+                }  /* if (g_adaptiveMode) */
+            }  /* if (READY_FOR_UPDATE()) */
+        }  /* while ((inBuff.pos != inBuff.size) */
     } while (directive != ZSTD_e_end);
 
     if (ferror(srcFile)) {

From 6b07a66aecbfff057397257456b69a148919e37e Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Wed, 19 Sep 2018 16:30:55 -0700
Subject: [PATCH 277/372] fixed minor reporting discrepancy in MT mode

---
 lib/compress/zstdmt_compress.c | 22 +++++++++++++++-------
 1 file changed, 15 insertions(+), 7 deletions(-)

diff --git a/lib/compress/zstdmt_compress.c b/lib/compress/zstdmt_compress.c
index 39255fdcf..a7e93dacd 100644
--- a/lib/compress/zstdmt_compress.c
+++ b/lib/compress/zstdmt_compress.c
@@ -712,7 +712,12 @@ void ZSTDMT_compressionJob(void* jobDescription)
         assert(job->cSize == 0);
         for (chunkNb = 1; chunkNb < nbChunks; chunkNb++) {
             size_t const cSize = ZSTD_compressContinue(cctx, op, oend-op, ip, chunkSize);
-            if (ZSTD_isError(cSize)) { job->cSize = cSize; goto _endJob; }
+            if (ZSTD_isError(cSize)) {
+                ZSTD_PTHREAD_MUTEX_LOCK(&job->job_mutex);
+                job->cSize = cSize;
+                ZSTD_pthread_mutex_unlock(&job->job_mutex);
+                goto _endJob;
+            }
             ip += chunkSize;
             op += cSize; assert(op < oend);
             /* stats */
@@ -725,7 +730,8 @@ void ZSTDMT_compressionJob(void* jobDescription)
             ZSTD_pthread_mutex_unlock(&job->job_mutex);
         }
         /* last block */
-        assert(chunkSize > 0); assert((chunkSize & (chunkSize - 1)) == 0);  /* chunkSize must be power of 2 for mask==(chunkSize-1) to work */
+        assert(chunkSize > 0);
+        assert((chunkSize & (chunkSize - 1)) == 0);  /* chunkSize must be power of 2 for mask==(chunkSize-1) to work */
         if ((nbChunks > 0) | job->lastJob /*must output a "last block" flag*/ ) {
             size_t const lastBlockSize1 = job->src.size & (chunkSize-1);
             size_t const lastBlockSize = ((lastBlockSize1==0) & (job->src.size>=chunkSize)) ? chunkSize : lastBlockSize1;
@@ -736,6 +742,7 @@ void ZSTDMT_compressionJob(void* jobDescription)
             /* stats */
             ZSTD_PTHREAD_MUTEX_LOCK(&job->job_mutex);
             job->cSize += cSize;
+            job->consumed = job->src.size;
             ZSTD_pthread_mutex_unlock(&job->job_mutex);
     }   }
 
@@ -748,10 +755,7 @@ _endJob:
     ZSTDMT_releaseSeq(job->seqPool, rawSeqStore);
     ZSTDMT_releaseCCtx(job->cctxPool, cctx);
     /* report */
-    ZSTD_PTHREAD_MUTEX_LOCK(&job->job_mutex);
-    job->consumed = job->src.size;
     ZSTD_pthread_cond_signal(&job->job_cond);
-    ZSTD_pthread_mutex_unlock(&job->job_mutex);
 }
 
 
@@ -1119,15 +1123,19 @@ size_t ZSTDMT_toFlushNow(ZSTDMT_CCtx* mtctx)
     assert(jobID <= mtctx->nextJobID);
     if (jobID == mtctx->nextJobID) return 0;   /* no active job => nothing to flush */
 
+    /* look into oldest non-fully-flushed job */
     {   unsigned const wJobID = jobID & mtctx->jobIDMask;
-        ZSTDMT_jobDescription* jobPtr = &mtctx->jobs[wJobID];
+        ZSTDMT_jobDescription* const jobPtr = &mtctx->jobs[wJobID];
         ZSTD_pthread_mutex_lock(&jobPtr->job_mutex);
         {   size_t const cResult = jobPtr->cSize;
             size_t const produced = ZSTD_isError(cResult) ? 0 : cResult;
             size_t const flushed = ZSTD_isError(cResult) ? 0 : jobPtr->dstFlushed;
             assert(flushed <= produced);
             toFlush = produced - flushed;
-            if (toFlush==0) assert(jobPtr->consumed < jobPtr->src.size);   /* if toFlush==0, doneJobID should still be active: if doneJobID is completed and fully flushed, ZSTDMT_flushProduced() should have already moved to next job */
+            if (toFlush==0 && (jobPtr->consumed >= jobPtr->src.size)) {
+                /* doneJobID is not-fully-flushed, but toFlush==0 : doneJobID should be compressing some more data */
+                assert(jobPtr->consumed < jobPtr->src.size);
+            }
         }
         ZSTD_pthread_mutex_unlock(&mtctx->jobs[wJobID].job_mutex);
     }

From 45010da074e60cd5cfca393ba8332442610688f2 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Wed, 19 Sep 2018 17:37:22 -0700
Subject: [PATCH 278/372] updated man page

and added `--adapt` test in `playTests.sh`
---
 programs/zstd.1    | 33 ++++++++++++----------
 programs/zstd.1.md |  8 ++++--
 tests/playTests.sh | 69 +++++++++++++++++++++++++---------------------
 3 files changed, 60 insertions(+), 50 deletions(-)

diff --git a/programs/zstd.1 b/programs/zstd.1
index 53fedbb9b..27ee6684c 100644
--- a/programs/zstd.1
+++ b/programs/zstd.1
@@ -1,5 +1,5 @@
 .
-.TH "ZSTD" "1" "2018-06-27" "zstd 1.3.5" "User Commands"
+.TH "ZSTD" "1" "September 2018" "zstd 1.3.5" "User Commands"
 .
 .SH "NAME"
 \fBzstd\fR \- zstd, zstdmt, unzstd, zstdcat \- Compress or decompress \.zst files
@@ -100,6 +100,10 @@ Display information related to a zstd compressed file, such as size, ratio, and
 \fB#\fR compression level [1\-19] (default: 3)
 .
 .TP
+\fB\-\-fast[=#]\fR
+switch to ultra\-fast compression levels\. If \fB=#\fR is not present, it defaults to \fB1\fR\. The higher the value, the faster the compression speed, at the cost of some compression ratio\. This setting overwrites compression level if one was set previously\. Similarly, if a compression level is set after \fB\-\-fast\fR, it overrides it\.
+.
+.TP
 \fB\-\-ultra\fR
 unlocks high compression levels 20+ (maximum 22), using a lot more memory\. Note that decompression will also require more memory when using these levels\.
 .
@@ -111,16 +115,16 @@ enables long distance matching with \fB#\fR \fBwindowLog\fR, if not \fB#\fR is n
 Note: If \fBwindowLog\fR is set to larger than 27, \fB\-\-long=windowLog\fR or \fB\-\-memory=windowSize\fR needs to be passed to the decompressor\.
 .
 .TP
-\fB\-\-fast[=#]\fR
-switch to ultra\-fast compression levels\. If \fB=#\fR is not present, it defaults to \fB1\fR\. The higher the value, the faster the compression speed, at the cost of some compression ratio\. This setting overwrites compression level if one was set previously\. Similarly, if a compression level is set after \fB\-\-fast\fR, it overrides it\.
-.
-.TP
 \fB\-T#\fR, \fB\-\-threads=#\fR
 Compress using \fB#\fR working threads (default: 1)\. If \fB#\fR is 0, attempt to detect and use the number of physical CPU cores\. In all cases, the nb of threads is capped to ZSTDMT_NBTHREADS_MAX==200\. This modifier does nothing if \fBzstd\fR is compiled without multithread support\.
 .
 .TP
 \fB\-\-single\-thread\fR
-Does not spawn a thread for compression, use caller thread instead\. This is the only available mode when multithread support is disabled\. In this mode, compression is serialized with I/O\. (This is different from \fB\-T1\fR, which spawns 1 compression thread in parallel of I/O)\. Single\-thread mode also features lower memory usage\.
+Does not spawn a thread for compression, use a single thread for both I/O and compression\. In this mode, compression is serialized with I/O, which is slightly slower\. (This is different from \fB\-T1\fR, which spawns 1 compression thread in parallel of I/O)\. This mode is the only one available when multithread support is disabled\. Single\-thread mode features lower memory usage\. Final compressed result is slightly different from \fB\-T1\fR\.
+.
+.TP
+\fB\-\-adapt\fR
+\fBzstd\fR will dynamically adapt compression level to perceived I/O conditions\. Compression level adaptation can be observed live by using command \fB\-v\fR\. The feature works when combined with multi\-threading and \fB\-\-long\fR mode\. It does not work with \fB\-\-single\-thread\fR\. It sets window size to 8 MB by default (can be changed manually, see \fBwlog\fR)\. Due to the chaotic nature of dynamic adaptation, compressed result is not reproducible\. \fInote\fR : at the time of this writing, \fB\-\-adapt\fR can remain stuck at low speed when combined with multiple worker threads (>=2)\.
 .
 .TP
 \fB\-D file\fR
@@ -194,7 +198,7 @@ All arguments after \fB\-\-\fR are treated as files
 Use FILEs as training set to create a dictionary\. The training set should contain a lot of small files (> 100), and weight typically 100x the target dictionary size (for example, 10 MB for a 100 KB dictionary)\.
 .
 .IP
-Supports multithreading if \fBzstd\fR is compiled with threading support\. Additional parameters can be specified with \fB\-\-train\-fastcover\fR\. The legacy dictionary builder can be accessed with \fB\-\-train\-legacy\fR\.  The cover dictionary builder can be accessed with \fB\-\-train\-cover\fR\. Equivalent to \fB\-\-train\-fastCover=d=8,steps=4\fR\.
+Supports multithreading if \fBzstd\fR is compiled with threading support\. Additional parameters can be specified with \fB\-\-train\-fastcover\fR\. The legacy dictionary builder can be accessed with \fB\-\-train\-legacy\fR\. The cover dictionary builder can be accessed with \fB\-\-train\-cover\fR\. Equivalent to \fB\-\-train\-fastcover=d=8,steps=4\fR\.
 .
 .TP
 \fB\-o file\fR
@@ -218,11 +222,10 @@ A dictionary ID is a locally unique ID that a decoder can use to verify it is us
 .
 .TP
 \fB\-\-train\-cover[=k#,d=#,steps=#,split=#]\fR
-Select parameters for the default dictionary builder algorithm named cover\. If \fId\fR is not specified, then it tries \fId\fR = 6 and \fId\fR = 8\. If \fIk\fR is not specified, then it tries \fIsteps\fR values in the range [50, 2000]\. If \fIsteps\fR is not specified, then the default value of 40 is used\. If \fIsplit\fR is not specified or \fIsplit\fR <= 0, then the default value of 100 is used\. If \fIsplit\fR is 100, all input samples are used for both training and testing
-to find optimal _d_ and _k_ to build dictionary.Requires that \fId\fR <= \fIk\fR\.
+Select parameters for the default dictionary builder algorithm named cover\. If \fId\fR is not specified, then it tries \fId\fR = 6 and \fId\fR = 8\. If \fIk\fR is not specified, then it tries \fIsteps\fR values in the range [50, 2000]\. If \fIsteps\fR is not specified, then the default value of 40 is used\. If \fIsplit\fR is not specified or split <= 0, then the default value of 100 is used\. Requires that \fId\fR <= \fIk\fR\.
 .
 .IP
-Selects segments of size \fIk\fR with highest score to put in the dictionary\. The score of a segment is computed by the sum of the frequencies of all the subsegments of size \fId\fR\. Generally \fId\fR should be in the range [6, 8], occasionally up to 16, but the algorithm will run faster with d <= \fI8\fR\. Good values for \fIk\fR vary widely based on the input data, but a safe range is [2 * \fId\fR, 2000]\. Supports multithreading if \fBzstd\fR is compiled with threading support\.
+Selects segments of size \fIk\fR with highest score to put in the dictionary\. The score of a segment is computed by the sum of the frequencies of all the subsegments of size \fId\fR\. Generally \fId\fR should be in the range [6, 8], occasionally up to 16, but the algorithm will run faster with d <= \fI8\fR\. Good values for \fIk\fR vary widely based on the input data, but a safe range is [2 * \fId\fR, 2000]\. If \fIsplit\fR is 100, all input samples are used for both training and testing to find optimal \fId\fR and \fIk\fR to build dictionary\. Supports multithreading if \fBzstd\fR is compiled with threading support\.
 .
 .IP
 Examples:
@@ -239,15 +242,15 @@ Examples:
 .IP
 \fBzstd \-\-train\-cover=k=50 FILEs\fR
 .
+.IP
+\fBzstd \-\-train\-cover=k=50,split=60 FILEs\fR
+.
 .TP
 \fB\-\-train\-fastcover[=k#,d=#,f=#,steps=#,split=#,accel=#]\fR
-Same as cover but with extra parameters \fIf\fR and \fIaccel\fR and different default value of split
+Same as cover but with extra parameters \fIf\fR and \fIaccel\fR and different default value of split If \fIsplit\fR is not specified, then it tries \fIsplit\fR = 75\. If \fIf\fR is not specified, then it tries \fIf\fR = 20\. Requires that 0 < \fIf\fR < 32\. If \fIaccel\fR is not specified, then it tries \fIaccel\fR = 1\. Requires that 0 < \fIaccel\fR <= 10\. Requires that \fId\fR = 6 or \fId\fR = 8\.
 .
 .IP
-If \fIsplit\fR is not specified, then it tries \fIsplit\fR = 75. If \fIf\fR is not specified, then it tries \fIf\fR = 20. Requires that 0 < \fIf\fR < 32. If \fIaccel\fR is not specified, then it tries \fIaccel\fR = 1. Requires that 0 < \fIaccel\fR <= 10. Requires that \fId\fR = 6 or \fId\fR = 8.
-.
-.IP
-\fIf\fR is log of size of array that keeps track of frequency of subsegments of size \fId\fR. The subsegment is hashed to an index in the range [0,2^\fIf\fR - 1]. It is possible that 2 different subsegments are hashed to the same index, and they are considered as the same subsegment when computing frequency. Using a higher \fIf\fR reduces collision but takes longer.
+\fIf\fR is log of size of array that keeps track of frequency of subsegments of size \fId\fR\. The subsegment is hashed to an index in the range [0,2^\fIf\fR \- 1]\. It is possible that 2 different subsegments are hashed to the same index, and they are considered as the same subsegment when computing frequency\. Using a higher \fIf\fR reduces collision but takes longer\.
 .
 .IP
 Examples:
diff --git a/programs/zstd.1.md b/programs/zstd.1.md
index 48d39474e..fcc1944bf 100644
--- a/programs/zstd.1.md
+++ b/programs/zstd.1.md
@@ -137,10 +137,12 @@ the last one takes effect.
 * `--adapt` :
     `zstd` will dynamically adapt compression level to perceived I/O conditions.
     Compression level adaptation can be observed live by using command `-v`.
-    Works with multi-threading and `--long` mode.
-    When not specified otherwise, uses a window size of 8 MB.
+    The feature works when combined with multi-threading and `--long` mode.
+    It does not work with `--single-thread`.
+    It sets window size to 8 MB by default (can be changed manually, see `wlog`).
     Due to the chaotic nature of dynamic adaptation, compressed result is not reproducible.
-    Adaptation does not work with `--single-thread`.
+    _note_ : at the time of this writing, `--adapt` can remain stuck at low speed
+    when combined with multiple worker threads (>=2).
 * `-D file`:
     use `file` as Dictionary to compress or decompress FILE(s)
 * `--no-dictID`:
diff --git a/tests/playTests.sh b/tests/playTests.sh
index 19b502999..4e543aaa0 100755
--- a/tests/playTests.sh
+++ b/tests/playTests.sh
@@ -417,38 +417,6 @@ test -f dictionary
 rm tmp* dictionary
 
 
-$ECHO "\n===>  cover dictionary builder : advanced options "
-
-TESTFILE=../programs/zstdcli.c
-./datagen > tmpDict
-$ECHO "- Create first dictionary"
-$ZSTD --train-cover=k=46,d=8,split=80 *.c ../programs/*.c -o tmpDict
-cp $TESTFILE tmp
-$ZSTD -f tmp -D tmpDict
-$ZSTD -d tmp.zst -D tmpDict -fo result
-$DIFF $TESTFILE result
-$ECHO "- Create second (different) dictionary"
-$ZSTD --train-cover=k=56,d=8 *.c ../programs/*.c ../programs/*.h -o tmpDictC
-$ZSTD -d tmp.zst -D tmpDictC -fo result && die "wrong dictionary not detected!"
-$ECHO "- Create dictionary with short dictID"
-$ZSTD --train-cover=k=46,d=8,split=80 *.c ../programs/*.c --dictID=1 -o tmpDict1
-cmp tmpDict tmpDict1 && die "dictionaries should have different ID !"
-$ECHO "- Create dictionary with size limit"
-$ZSTD --train-cover=steps=8 *.c ../programs/*.c -o tmpDict2 --maxdict=4K
-$ECHO "- Compare size of dictionary from 90% training samples with 80% training samples"
-$ZSTD --train-cover=split=90 -r *.c ../programs/*.c
-$ZSTD --train-cover=split=80 -r *.c ../programs/*.c
-$ECHO "- Create dictionary using all samples for both training and testing"
-$ZSTD --train-cover=split=100 -r *.c ../programs/*.c
-$ECHO "- Test -o before --train-cover"
-rm -f tmpDict dictionary
-$ZSTD -o tmpDict --train-cover *.c ../programs/*.c
-test -f tmpDict
-$ZSTD --train-cover *.c ../programs/*.c
-test -f dictionary
-rm tmp* dictionary
-
-
 $ECHO "\n===>  fastCover dictionary builder : advanced options "
 
 TESTFILE=../programs/zstdcli.c
@@ -890,6 +858,10 @@ roundTripTest -g700M -P50 "1 --single-thread --long=29"
 roundTripTest -g600M -P50 "1 --single-thread --long --zstd=wlog=29,clog=28"
 
 
+$ECHO "\n===>   adaptive mode "
+roundTripTest -g270000000 " --adapt"
+
+
 if [ -n "$hasMT" ]
 then
     $ECHO "\n===>   zstdmt long round-trip tests "
@@ -902,4 +874,37 @@ else
     $ECHO "\n**** no multithreading, skipping zstdmt tests **** "
 fi
 
+
+$ECHO "\n===>  cover dictionary builder : advanced options "
+
+TESTFILE=../programs/zstdcli.c
+./datagen > tmpDict
+$ECHO "- Create first dictionary"
+$ZSTD --train-cover=k=46,d=8,split=80 *.c ../programs/*.c -o tmpDict
+cp $TESTFILE tmp
+$ZSTD -f tmp -D tmpDict
+$ZSTD -d tmp.zst -D tmpDict -fo result
+$DIFF $TESTFILE result
+$ECHO "- Create second (different) dictionary"
+$ZSTD --train-cover=k=56,d=8 *.c ../programs/*.c ../programs/*.h -o tmpDictC
+$ZSTD -d tmp.zst -D tmpDictC -fo result && die "wrong dictionary not detected!"
+$ECHO "- Create dictionary with short dictID"
+$ZSTD --train-cover=k=46,d=8,split=80 *.c ../programs/*.c --dictID=1 -o tmpDict1
+cmp tmpDict tmpDict1 && die "dictionaries should have different ID !"
+$ECHO "- Create dictionary with size limit"
+$ZSTD --train-cover=steps=8 *.c ../programs/*.c -o tmpDict2 --maxdict=4K
+$ECHO "- Compare size of dictionary from 90% training samples with 80% training samples"
+$ZSTD --train-cover=split=90 -r *.c ../programs/*.c
+$ZSTD --train-cover=split=80 -r *.c ../programs/*.c
+$ECHO "- Create dictionary using all samples for both training and testing"
+$ZSTD --train-cover=split=100 -r *.c ../programs/*.c
+$ECHO "- Test -o before --train-cover"
+rm -f tmpDict dictionary
+$ZSTD -o tmpDict --train-cover *.c ../programs/*.c
+test -f tmpDict
+$ZSTD --train-cover *.c ../programs/*.c
+test -f dictionary
+rm tmp* dictionary
+
+
 rm tmp*

From 15519479ba2b6387952b95f9404329529e473581 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Thu, 20 Sep 2018 13:00:11 -0700
Subject: [PATCH 279/372] fixed minor gcc warning on a unused variable

---
 programs/zstdcli.c | 2 +-
 tests/playTests.sh | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/programs/zstdcli.c b/programs/zstdcli.c
index 36140c496..253bb0d06 100644
--- a/programs/zstdcli.c
+++ b/programs/zstdcli.c
@@ -1014,7 +1014,7 @@ int main(int argCount, const char* argv[])
         else
           operationResult = FIO_compressMultipleFilenames(filenameTable, filenameIdx, outFileName, suffix, dictFileName, cLevel, compressionParams);
 #else
-        (void)suffix;
+        (void)suffix; (void)adapt;
         DISPLAY("Compression not supported\n");
 #endif
     } else {  /* decompression or test */
diff --git a/tests/playTests.sh b/tests/playTests.sh
index 4e543aaa0..5898d183f 100755
--- a/tests/playTests.sh
+++ b/tests/playTests.sh
@@ -904,7 +904,7 @@ $ZSTD -o tmpDict --train-cover *.c ../programs/*.c
 test -f tmpDict
 $ZSTD --train-cover *.c ../programs/*.c
 test -f dictionary
-rm tmp* dictionary
+rm -f tmp* dictionary
 
 
-rm tmp*
+rm -f tmp*

From 7992942d6649df3bed2ce87dfb2d8889b60ce278 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Thu, 20 Sep 2018 13:47:31 -0700
Subject: [PATCH 280/372] fixed complex tsan issue

when job->consumed == job->src.size , compression job is presumed completed,
so it must be the very last action done in worker thread.
---
 lib/compress/zstdmt_compress.c | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/lib/compress/zstdmt_compress.c b/lib/compress/zstdmt_compress.c
index a7e93dacd..a7c409c62 100644
--- a/lib/compress/zstdmt_compress.c
+++ b/lib/compress/zstdmt_compress.c
@@ -640,6 +640,7 @@ void ZSTDMT_compressionJob(void* jobDescription)
     ZSTD_CCtx* const cctx = ZSTDMT_getCCtx(job->cctxPool);
     rawSeqStore_t rawSeqStore = ZSTDMT_getSeq(job->seqPool);
     buffer_t dstBuff = job->dstBuff;
+    size_t lastCBlockSize = 0;
 
     /* ressources */
     if (cctx==NULL) {
@@ -739,11 +740,7 @@ void ZSTDMT_compressionJob(void* jobDescription)
                  ZSTD_compressEnd     (cctx, op, oend-op, ip, lastBlockSize) :
                  ZSTD_compressContinue(cctx, op, oend-op, ip, lastBlockSize);
             if (ZSTD_isError(cSize)) { job->cSize = cSize; goto _endJob; }
-            /* stats */
-            ZSTD_PTHREAD_MUTEX_LOCK(&job->job_mutex);
-            job->cSize += cSize;
-            job->consumed = job->src.size;
-            ZSTD_pthread_mutex_unlock(&job->job_mutex);
+            lastCBlockSize = cSize;
     }   }
 
 _endJob:
@@ -755,7 +752,12 @@ _endJob:
     ZSTDMT_releaseSeq(job->seqPool, rawSeqStore);
     ZSTDMT_releaseCCtx(job->cctxPool, cctx);
     /* report */
+    ZSTD_PTHREAD_MUTEX_LOCK(&job->job_mutex);
+    if (ZSTD_isError(job->cSize)) assert(lastCBlockSize == 0);
+    job->cSize += lastCBlockSize;
+    job->consumed = job->src.size;  /* when job->consumed == job->src.size , compression job is presumed completed */
     ZSTD_pthread_cond_signal(&job->job_cond);
+    ZSTD_pthread_mutex_unlock(&job->job_mutex);
 }
 
 

From b2939163e10f7bb3f8de3a6721d8506941a43d7f Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Thu, 20 Sep 2018 14:24:23 -0700
Subject: [PATCH 281/372] Changed default legacy support to v0.5+

thus dropping read support for v0.4.

It's always possible to re-enable it, by changing build macro ZSTD_LEGACY_SUPPORT to 4.
---
 build/VS2008/zstd/zstd.vcproj                |  8 ++++----
 build/VS2008/zstdlib/zstdlib.vcproj          |  8 ++++----
 build/VS2010/libzstd-dll/libzstd-dll.vcxproj |  8 ++++----
 build/VS2010/libzstd/libzstd.vcxproj         |  8 ++++----
 build/VS2010/zstd/zstd.vcxproj               |  8 ++++----
 lib/Makefile                                 |  2 +-
 lib/README.md                                | 21 ++++++++++----------
 programs/Makefile                            |  2 +-
 8 files changed, 33 insertions(+), 32 deletions(-)

diff --git a/build/VS2008/zstd/zstd.vcproj b/build/VS2008/zstd/zstd.vcproj
index 46d7c85dd..595733d2b 100644
--- a/build/VS2008/zstd/zstd.vcproj
+++ b/build/VS2008/zstd/zstd.vcproj
@@ -45,7 +45,7 @@
 				Name="VCCLCompilerTool"
 				Optimization="0"
 				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\dictBuilder;$(SolutionDir)..\..\lib\compress"
-				PreprocessorDefinitions="ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=4;WIN32;_DEBUG;_CONSOLE"
+				PreprocessorDefinitions="ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=5;WIN32;_DEBUG;_CONSOLE"
 				MinimalRebuild="true"
 				BasicRuntimeChecks="3"
 				RuntimeLibrary="3"
@@ -122,7 +122,7 @@
 				EnableIntrinsicFunctions="true"
 				OmitFramePointers="true"
 				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\dictBuilder;$(SolutionDir)..\..\lib\compress"
-				PreprocessorDefinitions="ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=4;WIN32;NDEBUG;_CONSOLE"
+				PreprocessorDefinitions="ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=5;WIN32;NDEBUG;_CONSOLE"
 				RuntimeLibrary="0"
 				EnableFunctionLevelLinking="true"
 				UsePrecompiledHeader="0"
@@ -197,7 +197,7 @@
 				Name="VCCLCompilerTool"
 				Optimization="0"
 				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\dictBuilder;$(SolutionDir)..\..\lib\compress"
-				PreprocessorDefinitions="ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=4;WIN32;_DEBUG;_CONSOLE"
+				PreprocessorDefinitions="ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=5;WIN32;_DEBUG;_CONSOLE"
 				MinimalRebuild="true"
 				BasicRuntimeChecks="3"
 				RuntimeLibrary="3"
@@ -275,7 +275,7 @@
 				EnableIntrinsicFunctions="true"
 				OmitFramePointers="true"
 				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\dictBuilder;$(SolutionDir)..\..\lib\compress"
-				PreprocessorDefinitions="ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=4;WIN32;NDEBUG;_CONSOLE"
+				PreprocessorDefinitions="ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=5;WIN32;NDEBUG;_CONSOLE"
 				RuntimeLibrary="0"
 				EnableFunctionLevelLinking="true"
 				UsePrecompiledHeader="0"
diff --git a/build/VS2008/zstdlib/zstdlib.vcproj b/build/VS2008/zstdlib/zstdlib.vcproj
index 950a6f39c..5f89ebf82 100644
--- a/build/VS2008/zstdlib/zstdlib.vcproj
+++ b/build/VS2008/zstdlib/zstdlib.vcproj
@@ -45,7 +45,7 @@
 				Name="VCCLCompilerTool"
 				Optimization="0"
 				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\dictBuilder"
-				PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=4;WIN32;_DEBUG;_CONSOLE"
+				PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=5;WIN32;_DEBUG;_CONSOLE"
 				MinimalRebuild="true"
 				BasicRuntimeChecks="3"
 				RuntimeLibrary="3"
@@ -121,7 +121,7 @@
 				EnableIntrinsicFunctions="true"
 				OmitFramePointers="true"
 				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\dictBuilder"
-				PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=4;WIN32;NDEBUG;_CONSOLE"
+				PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=5;WIN32;NDEBUG;_CONSOLE"
 				RuntimeLibrary="0"
 				EnableFunctionLevelLinking="true"
 				UsePrecompiledHeader="0"
@@ -195,7 +195,7 @@
 				Name="VCCLCompilerTool"
 				Optimization="0"
 				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\dictBuilder"
-				PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=4;WIN32;_DEBUG;_CONSOLE"
+				PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=5;WIN32;_DEBUG;_CONSOLE"
 				MinimalRebuild="true"
 				BasicRuntimeChecks="3"
 				RuntimeLibrary="3"
@@ -272,7 +272,7 @@
 				EnableIntrinsicFunctions="true"
 				OmitFramePointers="true"
 				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\dictBuilder"
-				PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=4;WIN32;NDEBUG;_CONSOLE"
+				PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=5;WIN32;NDEBUG;_CONSOLE"
 				RuntimeLibrary="0"
 				EnableFunctionLevelLinking="true"
 				UsePrecompiledHeader="0"
diff --git a/build/VS2010/libzstd-dll/libzstd-dll.vcxproj b/build/VS2010/libzstd-dll/libzstd-dll.vcxproj
index 6e14e020e..e7e906e50 100644
--- a/build/VS2010/libzstd-dll/libzstd-dll.vcxproj
+++ b/build/VS2010/libzstd-dll/libzstd-dll.vcxproj
@@ -162,7 +162,7 @@
       
       Level4
       Disabled
-      ZSTD_DLL_EXPORT=1;ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=4;WIN32;_DEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)
+      ZSTD_DLL_EXPORT=1;ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=5;WIN32;_DEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)
       true
       EnableFastChecks
       MultiThreadedDebugDLL
@@ -182,7 +182,7 @@
       
       Level4
       Disabled
-      ZSTD_DLL_EXPORT=1;ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=4;WIN32;_DEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)
+      ZSTD_DLL_EXPORT=1;ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=5;WIN32;_DEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)
       true
       EnableFastChecks
       MultiThreadedDebugDLL
@@ -202,7 +202,7 @@
       MaxSpeed
       true
       true
-      ZSTD_DLL_EXPORT=1;ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=4;WIN32;NDEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)
+      ZSTD_DLL_EXPORT=1;ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=5;WIN32;NDEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)
       false
       MultiThreaded
       ProgramDatabase
@@ -224,7 +224,7 @@
       MaxSpeed
       true
       true
-      ZSTD_DLL_EXPORT=1;ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=4;WIN32;NDEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)
+      ZSTD_DLL_EXPORT=1;ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=5;WIN32;NDEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)
       false
       false
       MultiThreaded
diff --git a/build/VS2010/libzstd/libzstd.vcxproj b/build/VS2010/libzstd/libzstd.vcxproj
index 18f5cb533..7c2af2b7d 100644
--- a/build/VS2010/libzstd/libzstd.vcxproj
+++ b/build/VS2010/libzstd/libzstd.vcxproj
@@ -159,7 +159,7 @@
       
       Level4
       Disabled
-      ZSTD_DLL_EXPORT=1;ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=4;WIN32;_DEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)
+      ZSTD_DLL_EXPORT=1;ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=5;WIN32;_DEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)
       true
       EnableFastChecks
       MultiThreadedDebugDLL
@@ -179,7 +179,7 @@
       
       Level4
       Disabled
-      ZSTD_DLL_EXPORT=1;ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=4;WIN32;_DEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)
+      ZSTD_DLL_EXPORT=1;ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=5;WIN32;_DEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)
       true
       EnableFastChecks
       MultiThreadedDebugDLL
@@ -199,7 +199,7 @@
       MaxSpeed
       true
       true
-      ZSTD_DLL_EXPORT=1;ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=4;WIN32;NDEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)
+      ZSTD_DLL_EXPORT=1;ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=5;WIN32;NDEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)
       false
       MultiThreaded
       ProgramDatabase
@@ -221,7 +221,7 @@
       MaxSpeed
       true
       true
-      ZSTD_DLL_EXPORT=1;ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=4;WIN32;NDEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)
+      ZSTD_DLL_EXPORT=1;ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=5;WIN32;NDEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)
       false
       false
       MultiThreaded
diff --git a/build/VS2010/zstd/zstd.vcxproj b/build/VS2010/zstd/zstd.vcxproj
index 936960b58..aea18b247 100644
--- a/build/VS2010/zstd/zstd.vcxproj
+++ b/build/VS2010/zstd/zstd.vcxproj
@@ -169,7 +169,7 @@
       
       Level4
       Disabled
-      ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=4;WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)
+      ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=5;WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)
       true
       false
     
@@ -185,7 +185,7 @@
       
       Level4
       Disabled
-      ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=4;WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)
+      ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=5;WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)
       true
       false
     
@@ -203,7 +203,7 @@
       MaxSpeed
       true
       true
-      ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=4;WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)
+      ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=5;WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)
       false
       false
       MultiThreaded
@@ -224,7 +224,7 @@
       MaxSpeed
       true
       true
-      ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=4;WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)
+      ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=5;WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)
       false
       false
       MultiThreaded
diff --git a/lib/Makefile b/lib/Makefile
index cf8e45b0f..92f244dbe 100644
--- a/lib/Makefile
+++ b/lib/Makefile
@@ -38,7 +38,7 @@ ZDICT_FILES := $(sort $(wildcard dictBuilder/*.c))
 ZDEPR_FILES := $(sort $(wildcard deprecated/*.c))
 ZSTD_FILES := $(ZSTDCOMMON_FILES)
 
-ZSTD_LEGACY_SUPPORT ?= 4
+ZSTD_LEGACY_SUPPORT ?= 5
 ZSTD_LIB_COMPRESSION ?= 1
 ZSTD_LIB_DECOMPRESSION ?= 1
 ZSTD_LIB_DICTBUILDER ?= 1
diff --git a/lib/README.md b/lib/README.md
index 75debe872..0966c7aef 100644
--- a/lib/README.md
+++ b/lib/README.md
@@ -13,7 +13,7 @@ including commands variables, staged install, directory variables and standard t
 - `make install` : install libraries in default system directories
 
 `libzstd` default scope includes compression, decompression, dictionary building,
-and decoding support for legacy formats >= v0.4.0.
+and decoding support for legacy formats >= v0.5.0.
 
 
 #### API
@@ -48,23 +48,24 @@ It's possible to compile only a limited set of features.
         This module depends on both `lib/common` and `lib/compress` .
 - `lib/legacy` : source code to decompress legacy zstd formats, starting from `v0.1.0`.
         This module depends on `lib/common` and `lib/decompress`.
-        To enable this feature, it's required to define `ZSTD_LEGACY_SUPPORT` during compilation.
-        Typically, with `gcc`, add argument `-DZSTD_LEGACY_SUPPORT=1`.
-        Using higher number limits versions supported.
+        To enable this feature, define `ZSTD_LEGACY_SUPPORT` during compilation.
+        Specifying a number limits versions supported to that version onward.
         For example, `ZSTD_LEGACY_SUPPORT=2` means : "support legacy formats >= v0.2.0".
         `ZSTD_LEGACY_SUPPORT=3` means : "support legacy formats >= v0.3.0", and so on.
-        Starting v0.8.0, all versions of `zstd` produce frames compliant with specification.
-        As a consequence, `ZSTD_LEGACY_SUPPORT=8` (or more) doesn't trigger legacy support.
-        Also, `ZSTD_LEGACY_SUPPORT=0` means "do __not__ support legacy formats".
+        Currently, the default library setting is `ZST_LEGACY_SUPPORT=5`.
+        It can be changed at build by any other value.
+        Note that any number >= 8 translates into "do __not__ support legacy formats",
+        since all versions of `zstd` >= v0.8 are compatible with v1+ specification.
+        `ZSTD_LEGACY_SUPPORT=0` also means "do __not__ support legacy formats".
         Once enabled, this capability is transparently triggered within decompression functions.
         It's also possible to invoke directly legacy API, as exposed in `lib/legacy/zstd_legacy.h`.
         Each version also provides an additional dedicated set of advanced API.
         For example, advanced API for version `v0.4` is exposed in `lib/legacy/zstd_v04.h` .
         Note : `lib/legacy` only supports _decoding_ legacy formats.
-- Similarly, you can define `ZSTD_LIB_COMPRESSION, ZSTD_LIB_DECOMPRESSION`, `ZSTD_LIB_DICTBUILDER`, 
-        and `ZSTD_LIB_DEPRECATED` as 0 to forgo compilation of the corresponding features. This will 
+- Similarly, you can define `ZSTD_LIB_COMPRESSION, ZSTD_LIB_DECOMPRESSION`, `ZSTD_LIB_DICTBUILDER`,
+        and `ZSTD_LIB_DEPRECATED` as 0 to forgo compilation of the corresponding features. This will
         also disable compilation of all dependencies (eg. `ZSTD_LIB_COMPRESSION=0` will also disable
-        dictBuilder). 
+        dictBuilder).
 
 
 #### Multithreading support
diff --git a/programs/Makefile b/programs/Makefile
index f1a96325c..523aa5f76 100644
--- a/programs/Makefile
+++ b/programs/Makefile
@@ -58,7 +58,7 @@ ZSTD_FILES := $(ZSTDDECOMP_FILES) $(ZSTDCOMMON_FILES) $(ZSTDCOMP_FILES)
 ZDICT_FILES := $(ZSTDDIR)/dictBuilder/*.c
 ZSTDDECOMP_O = $(ZSTDDIR)/decompress/zstd_decompress.o
 
-ZSTD_LEGACY_SUPPORT ?= 4
+ZSTD_LEGACY_SUPPORT ?= 5
 ZSTDLEGACY_FILES :=
 ifneq ($(ZSTD_LEGACY_SUPPORT), 0)
 ifeq ($(shell test $(ZSTD_LEGACY_SUPPORT) -lt 8; echo $$?), 0)

From db97310ace8c8712c27c4f5b4833fdd9e1162f6f Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Thu, 20 Sep 2018 14:59:11 -0700
Subject: [PATCH 282/372] fixed versions-test to only test v0.5+

since zstd_devel is no longer compatible with v0.4+
---
 tests/test-zstd-versions.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/test-zstd-versions.py b/tests/test-zstd-versions.py
index f2deac1f2..8e88b869b 100755
--- a/tests/test-zstd-versions.py
+++ b/tests/test-zstd-versions.py
@@ -213,7 +213,7 @@ if __name__ == '__main__':
     print('Retrieve all release tags :')
     os.chdir(clone_dir)
     alltags = get_git_tags() + [head]
-    tags = [t for t in alltags if t >= 'v0.4.0']
+    tags = [t for t in alltags if t >= 'v0.5.0']
     print(tags)
 
     # Build all release zstd

From a54c86cfc62c2b273aab96a5e2a50c241ab42d46 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Thu, 20 Sep 2018 16:17:49 -0700
Subject: [PATCH 283/372] defined a minimum negative level

which can be probed using new function ZSTD_minCLevel().

Also : redefined ZSTD_TARGETLENGTH_MIN/MAX for consistency

used the opportunity to bump version number to v1.3.6
---
 doc/zstd_manual.html         | 67 ++++++++++++++++++------------------
 lib/compress/zstd_compress.c |  3 ++
 lib/zstd.h                   | 38 ++++++++++++--------
 tests/fuzzer.c               |  6 +++-
 4 files changed, 65 insertions(+), 49 deletions(-)

diff --git a/doc/zstd_manual.html b/doc/zstd_manual.html
index 5e01a19c3..6173712bf 100644
--- a/doc/zstd_manual.html
+++ b/doc/zstd_manual.html
@@ -1,10 +1,10 @@
 
 
 
-zstd 1.3.5 Manual
+zstd 1.3.6 Manual
 
 
-
zstd 1.3.5 Manual

+
zstd 1.3.6 Manual

 

 
Contents

 
    
@@ -18,20 +18,17 @@
 
  1. Streaming
    2. 
 
  2. Streaming compression - HowTo
    3. 
 
  3. Streaming decompression - HowTo
    4. 
-
  4. START OF ADVANCED AND EXPERIMENTAL FUNCTIONS
    5. 
-
  5. Advanced types
    6. 
-
  6. Frame size functions
    7. 
-
  7. ZSTD_frameHeaderSize() :
    8. 
-
  8. Memory management
    9. 
-
  9. Advanced compression functions
    10. 
-
  10. Advanced decompression functions
    11. 
-
  11. Advanced streaming functions
    12. 
-
  12. Buffer-less and synchronous inner streaming functions
    13. 
-
  13. Buffer-less streaming compression (synchronous mode)
    14. 
-
  14. Buffer-less streaming decompression (synchronous mode)
    15. 
-
  15. New advanced API (experimental)
    16. 
-
  16. ZSTD_getFrameHeader_advanced() :
    17. 
-
  17. Block level API
    18. 
+
  18. ADVANCED AND EXPERIMENTAL FUNCTIONS
    19. 
+
  19. Frame size functions
    20. 
+
  20. Memory management
    21. 
+
  21. Advanced compression functions
    22. 
+
  22. Advanced decompression functions
    23. 
+
  23. Advanced streaming functions
    24. 
+
  24. Buffer-less and synchronous inner streaming functions
    25. 
+
  25. Buffer-less streaming compression (synchronous mode)
    26. 
+
  26. Buffer-less streaming decompression (synchronous mode)
    27. 
+
  27. New advanced API (experimental)
    28. 
+
  28. Block level API
    29. 
 

 

 
Introduction
@@ -336,15 +333,16 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
 


 
size_t ZSTD_DStreamOutSize(void);   /*!< recommended size for output buffer. Guarantee to successfully flush at least one complete block in all circumstances. */
 


-
START OF ADVANCED AND EXPERIMENTAL FUNCTIONS
 The definitions in this section are considered experimental.
+
ADVANCED AND EXPERIMENTAL FUNCTIONS
+ The definitions in this section are considered experimental.
  They should never be used with a dynamic library, as prototypes may change in the future.
  They are provided for advanced scenarios.
  Use them only in association with static linking.
  
 

 
-
Advanced types

-
+
int ZSTD_minCLevel(void);  /*!< minimum negative compression level allowed */
+


 
typedef enum { ZSTD_fast=1, ZSTD_dfast, ZSTD_greedy, ZSTD_lazy, ZSTD_lazy2,
                ZSTD_btlazy2, ZSTD_btopt, ZSTD_btultra } ZSTD_strategy;   /* from faster to stronger */
 


@@ -380,7 +378,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
     ZSTD_dlm_byRef,      /**< Reference dictionary content -- the dictionary buffer must outlive its users. */
 } ZSTD_dictLoadMethod_e;
 


-
Frame size functions

+
Frame size functions

 
 
size_t ZSTD_findFrameCompressedSize(const void* src, size_t srcSize);
 
  `src` should point to the start of a ZSTD encoded frame or skippable frame
@@ -413,12 +411,13 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
             however it does mean that all frame data must be present and valid. 
 


 
-
ZSTD_frameHeaderSize() :
  srcSize must be >= ZSTD_frameHeaderSize_prefix.
+
size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize);
+
  srcSize must be >= ZSTD_frameHeaderSize_prefix.
  @return : size of the Frame Header,
            or an error code (if srcSize is too small) 
-

+


 
-
Memory management

+
Memory management

 
 
size_t ZSTD_sizeof_CCtx(const ZSTD_CCtx* cctx);
 size_t ZSTD_sizeof_DCtx(const ZSTD_DCtx* dctx);
@@ -508,7 +507,7 @@ static ZSTD_customMem const ZSTD_defaultCMem = { NULL, NULL, NULL };  /**< t
  
 


 
-
Advanced compression functions

+
Advanced compression functions

 
 
ZSTD_CDict* ZSTD_createCDict_byReference(const void* dictBuffer, size_t dictSize, int compressionLevel);
 
  Create a digested dictionary for compression
@@ -550,7 +549,7 @@ static ZSTD_customMem const ZSTD_defaultCMem = { NULL, NULL, NULL };  /**< t
 
   Same as ZSTD_compress_usingCDict(), with fine-tune control over frame parameters 
 


 
-
Advanced decompression functions

+
Advanced decompression functions

 
 
unsigned ZSTD_isFrame(const void* buffer, size_t size);
 
  Tells if the content of `buffer` starts with a valid Frame Identifier.
@@ -590,7 +589,7 @@ static ZSTD_customMem const ZSTD_defaultCMem = { NULL, NULL, NULL };  /**< t
   When identifying the exact failure cause, it's possible to use ZSTD_getFrameHeader(), which will provide a more precise error code. 
 


 
-
Advanced streaming functions

+
Advanced streaming functions

 
 
Advanced Streaming compression functions
size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pledgedSrcSize);   /**< pledgedSrcSize must be correct. If it is not known at init time, use ZSTD_CONTENTSIZE_UNKNOWN. Note that, for compatibility with older programs, "0" also disables frame content size field. It may be enabled in the future. */
 size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel); /**< creates of an internal CDict (incompatible with static CCtx), except if dict == NULL or dictSize < 8, in which case no dict is used. Note: dict is loaded with ZSTD_dm_auto (treated as a full zstd dictionary if it begins with ZSTD_MAGIC_DICTIONARY, else as raw content) and ZSTD_dlm_byCopy.*/
@@ -622,14 +621,14 @@ size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t di
 size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* zds, const ZSTD_DDict* ddict);  /**< note : ddict is referenced, it must outlive decompression session */
 size_t ZSTD_resetDStream(ZSTD_DStream* zds);  /**< re-use decompression parameters from previous init; saves dictionary loading */
 


-
Buffer-less and synchronous inner streaming functions
+
Buffer-less and synchronous inner streaming functions
   This is an advanced API, giving full control over buffer management, for users which need direct control over memory.
   But it's also a complex one, with several restrictions, documented below.
   Prefer normal streaming API for an easier experience.
  
 

 
-
Buffer-less streaming compression (synchronous mode)
+
Buffer-less streaming compression (synchronous mode)
   A ZSTD_CCtx object is required to track streaming operations.
   Use ZSTD_createCCtx() / ZSTD_freeCCtx() to manage resource.
   ZSTD_CCtx object can be re-used multiple times within successive compression operations.
@@ -665,7 +664,7 @@ size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict);
 size_t ZSTD_compressBegin_usingCDict_advanced(ZSTD_CCtx* const cctx, const ZSTD_CDict* const cdict, ZSTD_frameParameters const fParams, unsigned long long const pledgedSrcSize);   /* compression parameters are already set within cdict. pledgedSrcSize must be correct. If srcSize is not known, use macro ZSTD_CONTENTSIZE_UNKNOWN */
 size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned long long pledgedSrcSize); /**<  note: if pledgedSrcSize is not known, use ZSTD_CONTENTSIZE_UNKNOWN */
 


-
Buffer-less streaming decompression (synchronous mode)
+
Buffer-less streaming decompression (synchronous mode)
   A ZSTD_DCtx object is required to track streaming operations.
   Use ZSTD_createDCtx() / ZSTD_freeDCtx() to manage it.
   A ZSTD_DCtx object can be re-used multiple times.
@@ -756,7 +755,7 @@ size_t ZSTD_decodingBufferSize_min(unsigned long long windowSize, unsigned long
 


 
typedef enum { ZSTDnit_frameHeader, ZSTDnit_blockHeader, ZSTDnit_block, ZSTDnit_lastBlock, ZSTDnit_checksum, ZSTDnit_skippableFrame } ZSTD_nextInputType_e;
 


-
New advanced API (experimental)

+
New advanced API (experimental)

 
 
typedef enum {
     /* Opened question : should we have a format ZSTD_f_auto ?
@@ -1193,9 +1192,11 @@ size_t ZSTD_DCtx_refPrefix_advanced(ZSTD_DCtx* dctx,
  
 


 
-
ZSTD_getFrameHeader_advanced() :
  same as ZSTD_getFrameHeader(),
+
size_t ZSTD_getFrameHeader_advanced(ZSTD_frameHeader* zfhPtr,
+            const void* src, size_t srcSize, ZSTD_format_e format);
+
  same as ZSTD_getFrameHeader(),
   with added capability to select a format (like ZSTD_f_zstd1_magicless) 
-

+


 
 
size_t ZSTD_decompress_generic(ZSTD_DCtx* dctx,
                                ZSTD_outBuffer* output,
@@ -1229,7 +1230,7 @@ size_t ZSTD_DCtx_refPrefix_advanced(ZSTD_DCtx* dctx,
  
 


 
-
Block level API

+
Block level API

 
 
    Frame metadata cost is typically ~18 bytes, which can be non-negligible for very small blocks (< 100 bytes).
     User will have to take in charge required information to regenerate data, such as compressed and content sizes.
diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index f375ee2ce..2862a6bd8 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -679,6 +679,7 @@ size_t ZSTD_checkCParams(ZSTD_compressionParameters cParams)
     CLAMPCHECK(cParams.hashLog, ZSTD_HASHLOG_MIN, ZSTD_HASHLOG_MAX);
     CLAMPCHECK(cParams.searchLog, ZSTD_SEARCHLOG_MIN, ZSTD_SEARCHLOG_MAX);
     CLAMPCHECK(cParams.searchLength, ZSTD_SEARCHLENGTH_MIN, ZSTD_SEARCHLENGTH_MAX);
+    CLAMPCHECK(cParams.targetLength, ZSTD_TARGETLENGTH_MIN, ZSTD_TARGETLENGTH_MAX);
     if ((U32)(cParams.strategy) > (U32)ZSTD_btultra)
         return ERROR(parameter_unsupported);
     return 0;
@@ -699,6 +700,7 @@ ZSTD_clampCParams(ZSTD_compressionParameters cParams)
     CLAMP(cParams.hashLog, ZSTD_HASHLOG_MIN, ZSTD_HASHLOG_MAX);
     CLAMP(cParams.searchLog, ZSTD_SEARCHLOG_MIN, ZSTD_SEARCHLOG_MAX);
     CLAMP(cParams.searchLength, ZSTD_SEARCHLENGTH_MIN, ZSTD_SEARCHLENGTH_MAX);
+    CLAMP(cParams.targetLength, ZSTD_TARGETLENGTH_MIN, ZSTD_TARGETLENGTH_MAX);
     CLAMP(cParams.strategy, ZSTD_fast, ZSTD_btultra);
     return cParams;
 }
@@ -3782,6 +3784,7 @@ size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output)
 
 #define ZSTD_MAX_CLEVEL     22
 int ZSTD_maxCLevel(void) { return ZSTD_MAX_CLEVEL; }
+int ZSTD_minCLevel(void) { return (int)-ZSTD_TARGETLENGTH_MAX; }
 
 static const ZSTD_compressionParameters ZSTD_defaultCParameters[4][ZSTD_MAX_CLEVEL+1] = {
 {   /* "default" - guarantees a monotonically increasing memory budget */
diff --git a/lib/zstd.h b/lib/zstd.h
index 277ab7ffb..60260d2bd 100644
--- a/lib/zstd.h
+++ b/lib/zstd.h
@@ -71,7 +71,7 @@ extern "C" {
 /*------   Version   ------*/
 #define ZSTD_VERSION_MAJOR    1
 #define ZSTD_VERSION_MINOR    3
-#define ZSTD_VERSION_RELEASE  5
+#define ZSTD_VERSION_RELEASE  6
 
 #define ZSTD_VERSION_NUMBER  (ZSTD_VERSION_MAJOR *100*100 + ZSTD_VERSION_MINOR *100 + ZSTD_VERSION_RELEASE)
 ZSTDLIB_API unsigned ZSTD_versionNumber(void);   /**< useful to check dll version */
@@ -80,7 +80,7 @@ ZSTDLIB_API unsigned ZSTD_versionNumber(void);   /**< useful to check dll versio
 #define ZSTD_QUOTE(str) #str
 #define ZSTD_EXPAND_AND_QUOTE(str) ZSTD_QUOTE(str)
 #define ZSTD_VERSION_STRING ZSTD_EXPAND_AND_QUOTE(ZSTD_LIB_VERSION)
-ZSTDLIB_API const char* ZSTD_versionString(void);   /* added in v1.3.0 */
+ZSTDLIB_API const char* ZSTD_versionString(void);   /* v1.3.0+ */
 
 /***************************************
 *  Default constant
@@ -330,7 +330,7 @@ typedef struct ZSTD_outBuffer_s {
 * *******************************************************************/
 
 typedef ZSTD_CCtx ZSTD_CStream;  /**< CCtx and CStream are now effectively same object (>= v1.3.0) */
-                                 /* Continue to distinguish them for compatibility with versions <= v1.2.0 */
+                                 /* Continue to distinguish them for compatibility with older versions <= v1.2.0 */
 /*===== ZSTD_CStream management functions =====*/
 ZSTDLIB_API ZSTD_CStream* ZSTD_createCStream(void);
 ZSTDLIB_API size_t ZSTD_freeCStream(ZSTD_CStream* zcs);
@@ -385,21 +385,28 @@ ZSTDLIB_API size_t ZSTD_DStreamOutSize(void);   /*!< recommended size for output
 
 
 
+
+#if defined(ZSTD_STATIC_LINKING_ONLY) && !defined(ZSTD_H_ZSTD_STATIC_LINKING_ONLY)
+#define ZSTD_H_ZSTD_STATIC_LINKING_ONLY
+
 /****************************************************************************************
- * START OF ADVANCED AND EXPERIMENTAL FUNCTIONS
+ *   ADVANCED AND EXPERIMENTAL FUNCTIONS
+ ****************************************************************************************
  * The definitions in this section are considered experimental.
  * They should never be used with a dynamic library, as prototypes may change in the future.
  * They are provided for advanced scenarios.
  * Use them only in association with static linking.
  * ***************************************************************************************/
 
-#if defined(ZSTD_STATIC_LINKING_ONLY) && !defined(ZSTD_H_ZSTD_STATIC_LINKING_ONLY)
-#define ZSTD_H_ZSTD_STATIC_LINKING_ONLY
+ZSTDLIB_API int ZSTD_minCLevel(void);  /*!< minimum negative compression level allowed */
 
-/* --- Constants ---*/
-#define ZSTD_MAGICNUMBER            0xFD2FB528   /* >= v0.8.0 */
+/* ---  Constants  ---*/
+#define ZSTD_MAGICNUMBER            0xFD2FB528   /* v0.8+ */
+#define ZSTD_MAGIC_DICTIONARY       0xEC30A437   /* v0.7+ */
 #define ZSTD_MAGIC_SKIPPABLE_START  0x184D2A50U
-#define ZSTD_MAGIC_DICTIONARY       0xEC30A437   /* >= v0.7.0 */
+
+#define ZSTD_BLOCKSIZELOG_MAX 17
+#define ZSTD_BLOCKSIZE_MAX   (1<= ZSTD_frameHeaderSize_prefix.
  * @return : size of the Frame Header,
  *           or an error code (if srcSize is too small) */
@@ -1401,7 +1411,7 @@ ZSTDLIB_API size_t ZSTD_DCtx_setMaxWindowSize(ZSTD_DCtx* dctx, size_t maxWindowS
 ZSTDLIB_API size_t ZSTD_DCtx_setFormat(ZSTD_DCtx* dctx, ZSTD_format_e format);
 
 
-/** ZSTD_getFrameHeader_advanced() :
+/*! ZSTD_getFrameHeader_advanced() :
  *  same as ZSTD_getFrameHeader(),
  *  with added capability to select a format (like ZSTD_f_zstd1_magicless) */
 ZSTDLIB_API size_t ZSTD_getFrameHeader_advanced(ZSTD_frameHeader* zfhPtr,
@@ -1473,8 +1483,6 @@ ZSTDLIB_API void ZSTD_DCtx_reset(ZSTD_DCtx* dctx);
         Use ZSTD_insertBlock() for such a case.
 */
 
-#define ZSTD_BLOCKSIZELOG_MAX 17
-#define ZSTD_BLOCKSIZE_MAX   (1<
Date: Fri, 21 Sep 2018 14:46:09 -0700
Subject: [PATCH 284/372] fix mingw compatibility

only enable backtraces for platforms we know support it
aka mac OS-X and Linux.
can be extended later.
---
 programs/fileio.c | 12 ++++++++----
 programs/util.h   |  6 +++++-
 2 files changed, 13 insertions(+), 5 deletions(-)

diff --git a/programs/fileio.c b/programs/fileio.c
index 5f6b12e51..e08920ba8 100644
--- a/programs/fileio.c
+++ b/programs/fileio.c
@@ -20,6 +20,9 @@
 #  define _POSIX_SOURCE 1          /* disable %llu warnings with MinGW on Windows */
 #endif
 
+#if  defined(__linux__) || (defined(__APPLE__) && defined(__MACH__))
+#  define BACKTRACES_ENABLE 1
+#endif
 
 /*-*************************************
 *  Includes
@@ -31,8 +34,8 @@
 #include      /* strcmp, strlen */
 #include       /* errno */
 #include 
-#ifndef _WIN32
-#include    /* backtrace, backtrace_symbols */
+#ifdef BACKTRACES_ENABLE
+#  include    /* backtrace, backtrace_symbols */
 #endif
 
 #if defined (_MSC_VER)
@@ -162,9 +165,10 @@ static void clearHandler(void)
 /*-*********************************************************
 *  Termination signal trapping (Print debug stack trace)
 ***********************************************************/
+#ifdef BACKTRACES_ENABLE
+
 #define MAX_STACK_FRAMES    50
 
-#ifndef _WIN32
 static void ABRThandler(int sig) {
     const char* name;
     void* addrlist[MAX_STACK_FRAMES];
@@ -202,7 +206,7 @@ static void ABRThandler(int sig) {
 
 void FIO_addAbortHandler()
 {
-#ifndef _WIN32
+#ifdef BACKTRACES_ENABLE
     signal(SIGABRT, ABRThandler);
     signal(SIGFPE, ABRThandler);
     signal(SIGILL, ABRThandler);
diff --git a/programs/util.h b/programs/util.h
index 8993bf964..e8288b8fe 100644
--- a/programs/util.h
+++ b/programs/util.h
@@ -170,7 +170,11 @@ static int g_utilDisplayLevel;
         return ((clockEnd - clockStart) * (U64)rate.numer) / ((U64)rate.denom);
     }
 
-#elif (PLATFORM_POSIX_VERSION >= 200112L) && (defined __UCLIBC__ || ((__GLIBC__ == 2 && __GLIBC_MINOR__ >= 17) || __GLIBC__ > 2))
+#elif (PLATFORM_POSIX_VERSION >= 200112L) \
+   && (defined(__UCLIBC__)                \
+      || (defined(__GLIBC__)              \
+          && ((__GLIBC__ == 2 && __GLIBC_MINOR__ >= 17) \
+             || (__GLIBC__ > 2))))
 
     #define UTIL_TIME_INITIALIZER { 0, 0 }
     typedef struct timespec UTIL_freq_t;

From 32b7cf1bcfbe14a5f8fa3569e16302ba4603d82e Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Fri, 21 Sep 2018 15:04:43 -0700
Subject: [PATCH 285/372] fixed tautological tests

involving ZSTD_TARGETLENGTH_MIN (== 0)
---
 lib/compress/zstd_compress.c |   8 ++++++--
 lib/zstd.h                   |   2 +-
 programs/windres/zstd32.res  | Bin 1044 -> 1044 bytes
 programs/windres/zstd64.res  | Bin 1044 -> 1044 bytes
 4 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index 2862a6bd8..98e9917ab 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -679,7 +679,9 @@ size_t ZSTD_checkCParams(ZSTD_compressionParameters cParams)
     CLAMPCHECK(cParams.hashLog, ZSTD_HASHLOG_MIN, ZSTD_HASHLOG_MAX);
     CLAMPCHECK(cParams.searchLog, ZSTD_SEARCHLOG_MIN, ZSTD_SEARCHLOG_MAX);
     CLAMPCHECK(cParams.searchLength, ZSTD_SEARCHLENGTH_MIN, ZSTD_SEARCHLENGTH_MAX);
-    CLAMPCHECK(cParams.targetLength, ZSTD_TARGETLENGTH_MIN, ZSTD_TARGETLENGTH_MAX);
+    ZSTD_STATIC_ASSERT(ZSTD_TARGETLENGTH_MIN == 0);
+    if (cParams.targetLength > ZSTD_TARGETLENGTH_MAX)
+        return ERROR(parameter_outOfBound);
     if ((U32)(cParams.strategy) > (U32)ZSTD_btultra)
         return ERROR(parameter_unsupported);
     return 0;
@@ -700,7 +702,9 @@ ZSTD_clampCParams(ZSTD_compressionParameters cParams)
     CLAMP(cParams.hashLog, ZSTD_HASHLOG_MIN, ZSTD_HASHLOG_MAX);
     CLAMP(cParams.searchLog, ZSTD_SEARCHLOG_MIN, ZSTD_SEARCHLOG_MAX);
     CLAMP(cParams.searchLength, ZSTD_SEARCHLENGTH_MIN, ZSTD_SEARCHLENGTH_MAX);
-    CLAMP(cParams.targetLength, ZSTD_TARGETLENGTH_MIN, ZSTD_TARGETLENGTH_MAX);
+    ZSTD_STATIC_ASSERT(ZSTD_TARGETLENGTH_MIN == 0);
+    if (cParams.targetLength > ZSTD_TARGETLENGTH_MAX)
+        cParams.targetLength = ZSTD_TARGETLENGTH_MAX;
     CLAMP(cParams.strategy, ZSTD_fast, ZSTD_btultra);
     return cParams;
 }
diff --git a/lib/zstd.h b/lib/zstd.h
index 60260d2bd..8a39de59a 100644
--- a/lib/zstd.h
+++ b/lib/zstd.h
@@ -424,7 +424,7 @@ ZSTDLIB_API int ZSTD_minCLevel(void);  /*!< minimum negative compression level a
 #define ZSTD_SEARCHLENGTH_MAX    7   /* only for ZSTD_fast, other strategies are limited to 6 */
 #define ZSTD_SEARCHLENGTH_MIN    3   /* only for ZSTD_btopt, other strategies are limited to 4 */
 #define ZSTD_TARGETLENGTH_MAX  ZSTD_BLOCKSIZE_MAX
-#define ZSTD_TARGETLENGTH_MIN    0
+#define ZSTD_TARGETLENGTH_MIN    0   /* note : comparing this constant to an unsigned results in a tautological test */
 #define ZSTD_LDM_MINMATCH_MAX 4096
 #define ZSTD_LDM_MINMATCH_MIN    4
 #define ZSTD_LDM_BUCKETSIZELOG_MAX 8
diff --git a/programs/windres/zstd32.res b/programs/windres/zstd32.res
index 26800ab321c2fc6b0a0d57a28e2710f71c8732c7..276cb20b7871cc800fb8f6f7b792520c5e6e7957 100644
GIT binary patch
delta 33
pcmbQjF@`O`PF~5J4FH7X2oL}O

diff --git a/programs/windres/zstd64.res b/programs/windres/zstd64.res
index 723d73a612e0df1bf8aa2a66541a4ad2ea5696f0..3eb0162f01a76591ac25bd5049ebc06d1cd510aa 100644
GIT binary patch
delta 33
pcmbQjF@`O`PF~5J4FH7X2oL}O


From bfff4f4809777f114888db858ccd4aa2c18641bb Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Fri, 21 Sep 2018 15:37:30 -0700
Subject: [PATCH 286/372] ensure all writes to job->cSize are mutex protected

even when reporting errors,
using a macro for code brevity, as suggested by @terrelln,
---
 lib/compress/zstdmt_compress.c | 52 ++++++++++++++--------------------
 programs/fileio.c              |  3 ++
 2 files changed, 24 insertions(+), 31 deletions(-)

diff --git a/lib/compress/zstdmt_compress.c b/lib/compress/zstdmt_compress.c
index a7c409c62..6b9c24b56 100644
--- a/lib/compress/zstdmt_compress.c
+++ b/lib/compress/zstdmt_compress.c
@@ -632,6 +632,13 @@ typedef struct {
     unsigned frameChecksumNeeded;        /* used only by mtctx */
 } ZSTDMT_jobDescription;
 
+#define JOB_ERROR(e) {                          \
+    ZSTD_PTHREAD_MUTEX_LOCK(&job->job_mutex);   \
+    job->cSize = e;                             \
+    ZSTD_pthread_mutex_unlock(&job->job_mutex); \
+    goto _endJob;                               \
+}
+
 /* ZSTDMT_compressionJob() is a POOL_function type */
 void ZSTDMT_compressionJob(void* jobDescription)
 {
@@ -643,22 +650,14 @@ void ZSTDMT_compressionJob(void* jobDescription)
     size_t lastCBlockSize = 0;
 
     /* ressources */
-    if (cctx==NULL) {
-        job->cSize = ERROR(memory_allocation);
-        goto _endJob;
-    }
+    if (cctx==NULL) JOB_ERROR(ERROR(memory_allocation));
     if (dstBuff.start == NULL) {   /* streaming job : doesn't provide a dstBuffer */
         dstBuff = ZSTDMT_getBuffer(job->bufPool);
-        if (dstBuff.start==NULL) {
-            job->cSize = ERROR(memory_allocation);
-            goto _endJob;
-        }
+        if (dstBuff.start==NULL) JOB_ERROR(ERROR(memory_allocation));
         job->dstBuff = dstBuff;   /* this value can be read in ZSTDMT_flush, when it copies the whole job */
     }
-    if (jobParams.ldmParams.enableLdm && rawSeqStore.seq == NULL) {
-        job->cSize = ERROR(memory_allocation);
-        goto _endJob;
-    }
+    if (jobParams.ldmParams.enableLdm && rawSeqStore.seq == NULL)
+        JOB_ERROR(ERROR(memory_allocation));
 
     /* Don't compute the checksum for chunks, since we compute it externally,
      * but write it in the header.
@@ -672,30 +671,26 @@ void ZSTDMT_compressionJob(void* jobDescription)
     if (job->cdict) {
         size_t const initError = ZSTD_compressBegin_advanced_internal(cctx, NULL, 0, ZSTD_dct_auto, ZSTD_dtlm_fast, job->cdict, jobParams, job->fullFrameSize);
         assert(job->firstJob);  /* only allowed for first job */
-        if (ZSTD_isError(initError)) { job->cSize = initError; goto _endJob; }
+        if (ZSTD_isError(initError)) JOB_ERROR(initError);
     } else {  /* srcStart points at reloaded section */
         U64 const pledgedSrcSize = job->firstJob ? job->fullFrameSize : job->src.size;
         {   size_t const forceWindowError = ZSTD_CCtxParam_setParameter(&jobParams, ZSTD_p_forceMaxWindow, !job->firstJob);
-            if (ZSTD_isError(forceWindowError)) {
-                job->cSize = forceWindowError;
-                goto _endJob;
-        }   }
+            if (ZSTD_isError(forceWindowError)) JOB_ERROR(forceWindowError);
+        }
         {   size_t const initError = ZSTD_compressBegin_advanced_internal(cctx,
                                         job->prefix.start, job->prefix.size, ZSTD_dct_rawContent, /* load dictionary in "content-only" mode (no header analysis) */
                                         ZSTD_dtlm_fast,
                                         NULL, /*cdict*/
                                         jobParams, pledgedSrcSize);
-            if (ZSTD_isError(initError)) {
-                job->cSize = initError;
-                goto _endJob;
-    }   }   }
+            if (ZSTD_isError(initError)) JOB_ERROR(initError);
+    }   }
 
     /* Perform serial step as early as possible, but after CCtx initialization */
     ZSTDMT_serialState_update(job->serial, cctx, rawSeqStore, job->src, job->jobID);
 
     if (!job->firstJob) {  /* flush and overwrite frame header when it's not first job */
         size_t const hSize = ZSTD_compressContinue(cctx, dstBuff.start, dstBuff.capacity, job->src.start, 0);
-        if (ZSTD_isError(hSize)) { job->cSize = hSize; /* save error code */ goto _endJob; }
+        if (ZSTD_isError(hSize)) JOB_ERROR(hSize);
         DEBUGLOG(5, "ZSTDMT_compressionJob: flush and overwrite %u bytes of frame header (not first job)", (U32)hSize);
         ZSTD_invalidateRepCodes(cctx);
     }
@@ -713,12 +708,7 @@ void ZSTDMT_compressionJob(void* jobDescription)
         assert(job->cSize == 0);
         for (chunkNb = 1; chunkNb < nbChunks; chunkNb++) {
             size_t const cSize = ZSTD_compressContinue(cctx, op, oend-op, ip, chunkSize);
-            if (ZSTD_isError(cSize)) {
-                ZSTD_PTHREAD_MUTEX_LOCK(&job->job_mutex);
-                job->cSize = cSize;
-                ZSTD_pthread_mutex_unlock(&job->job_mutex);
-                goto _endJob;
-            }
+            if (ZSTD_isError(cSize)) JOB_ERROR(cSize);
             ip += chunkSize;
             op += cSize; assert(op < oend);
             /* stats */
@@ -739,7 +729,7 @@ void ZSTDMT_compressionJob(void* jobDescription)
             size_t const cSize = (job->lastJob) ?
                  ZSTD_compressEnd     (cctx, op, oend-op, ip, lastBlockSize) :
                  ZSTD_compressContinue(cctx, op, oend-op, ip, lastBlockSize);
-            if (ZSTD_isError(cSize)) { job->cSize = cSize; goto _endJob; }
+            if (ZSTD_isError(cSize)) JOB_ERROR(cSize);
             lastCBlockSize = cSize;
     }   }
 
@@ -1657,7 +1647,7 @@ static size_t ZSTDMT_flushProduced(ZSTDMT_CCtx* mtctx, ZSTD_outBuffer* output, u
                 DEBUGLOG(5, "Job %u completed (%u bytes), moving to next one",
                         mtctx->doneJobID, (U32)mtctx->jobs[wJobID].dstFlushed);
                 ZSTDMT_releaseBuffer(mtctx->bufPool, mtctx->jobs[wJobID].dstBuff);
-                DEBUGLOG(5, "dstBuffer released")
+                DEBUGLOG(5, "dstBuffer released");
                 mtctx->jobs[wJobID].dstBuff = g_nullBuffer;
                 mtctx->jobs[wJobID].cSize = 0;   /* ensure this job slot is considered "not started" in future check */
                 mtctx->consumed += srcSize;
@@ -1880,7 +1870,7 @@ size_t ZSTDMT_compressStream_generic(ZSTDMT_CCtx* mtctx,
                 /* It is only possible for this operation to fail if there are
                  * still compression jobs ongoing.
                  */
-                DEBUGLOG(5, "ZSTDMT_tryGetInputRange failed")
+                DEBUGLOG(5, "ZSTDMT_tryGetInputRange failed");
                 assert(mtctx->doneJobID != mtctx->nextJobID);
             } else
                 DEBUGLOG(5, "ZSTDMT_tryGetInputRange completed successfully : mtctx->inBuff.buffer.start = %p", mtctx->inBuff.buffer.start);
diff --git a/programs/fileio.c b/programs/fileio.c
index 00f0bc263..f3800b689 100644
--- a/programs/fileio.c
+++ b/programs/fileio.c
@@ -892,6 +892,9 @@ FIO_compressZstdFrame(const cRess_t* ressPtr,
                         assert(zfp.produced >= previous_zfp_update.produced);
                         assert(g_nbWorkers >= 1);
 
+                        /* test if output speed is so slow that all buffers are full
+                         * and no further progress is possible
+                         * (neither compression nor adding more input into internal buffers) */
                         if ( (zfp.ingested == previous_zfp_update.ingested)   /* no data read : input buffer full */
                           && (zfp.consumed == previous_zfp_update.consumed)   /* no data compressed : no more buffer to compress OR compression is really slow */
                           && (zfp.nbActiveWorkers == 0)          /* confirmed : no compression : either no more buffer to compress OR not enough data to start first worker */

From 00c18c0c884370f434a212e52ba425db8b70de80 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Fri, 21 Sep 2018 16:35:43 -0700
Subject: [PATCH 287/372] simplified "slows down when compression blocked"

---
 programs/fileio.c | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/programs/fileio.c b/programs/fileio.c
index 407680eff..964870f39 100644
--- a/programs/fileio.c
+++ b/programs/fileio.c
@@ -896,12 +896,12 @@ FIO_compressZstdFrame(const cRess_t* ressPtr,
                         assert(zfp.produced >= previous_zfp_update.produced);
                         assert(g_nbWorkers >= 1);
 
-                        /* test if output speed is so slow that all buffers are full
-                         * and no further progress is possible
-                         * (neither compression nor adding more input into internal buffers) */
-                        if ( (zfp.ingested == previous_zfp_update.ingested)   /* no data read : input buffer full */
-                          && (zfp.consumed == previous_zfp_update.consumed)   /* no data compressed : no more buffer to compress OR compression is really slow */
-                          && (zfp.nbActiveWorkers == 0)          /* confirmed : no compression : either no more buffer to compress OR not enough data to start first worker */
+                        /* test if compression is blocked
+                         * either because output is slow and all buffers are full
+                         * or because input is slow and no job can start while waiting for at least one buffer to be filled.
+                         * note : excluse starting part, since currentJobID > 1 */
+                        if ( (zfp.consumed == previous_zfp_update.consumed)   /* no data compressed : no data available, or no more buffer to compress to, OR compression is really slow (compression of a single block is slower than update rate)*/
+                          && (zfp.nbActiveWorkers == 0)                       /* confirmed : no compression ongoing */
                           ) {
                             DISPLAYLEVEL(6, "all buffers full : compression stopped => slow down \n")
                             speedChange = slower;

From 484f40697b2680c6fcb7e6d4d36975fb421c44ae Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Fri, 21 Sep 2018 17:28:37 -0700
Subject: [PATCH 288/372] fix constant redeclaration in paramgrill

---
 tests/paramgrill.c | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/tests/paramgrill.c b/tests/paramgrill.c
index 5275f10a5..dbddd242f 100644
--- a/tests/paramgrill.c
+++ b/tests/paramgrill.c
@@ -70,9 +70,6 @@ static const int g_maxNbVariations = 64;
 #define FADT_MIN 0
 #define FADT_MAX ((U32)-1)
 
-#define ZSTD_TARGETLENGTH_MIN 0
-#define ZSTD_TARGETLENGTH_MAX 999
-
 #define WLOG_RANGE (ZSTD_WINDOWLOG_MAX - ZSTD_WINDOWLOG_MIN + 1)
 #define CLOG_RANGE (ZSTD_CHAINLOG_MAX - ZSTD_CHAINLOG_MIN + 1)
 #define HLOG_RANGE (ZSTD_HASHLOG_MAX - ZSTD_HASHLOG_MIN + 1)
@@ -1192,6 +1189,7 @@ static int createContexts(contexts_t* ctx, const char* dictFileName) {
         return 0;
     }
     ctx->dictSize = UTIL_getFileSize(dictFileName);
+    assert(ctx->dictSize != UTIL_FILESIZE_UNKNOWN);
     ctx->dictBuffer = malloc(ctx->dictSize);
 
     f = fopen(dictFileName, "rb");

From 123fac6b6dd168b9ed8c4844eba9080d7084acb1 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Fri, 21 Sep 2018 17:36:00 -0700
Subject: [PATCH 289/372] fix pzstd compatibility with mingw

some details changed with introduction of gcc7
---
 contrib/pzstd/Options.cpp | 11 -----------
 contrib/pzstd/Pzstd.cpp   |  9 +--------
 zlibWrapper/gzguts.h      |  4 ++--
 3 files changed, 3 insertions(+), 21 deletions(-)

diff --git a/contrib/pzstd/Options.cpp b/contrib/pzstd/Options.cpp
index 1590d85ee..2123f8894 100644
--- a/contrib/pzstd/Options.cpp
+++ b/contrib/pzstd/Options.cpp
@@ -18,17 +18,6 @@
 #include 
 #include 
 
-#if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) ||     \
-    defined(__CYGWIN__)
-#include  /* _isatty */
-#define IS_CONSOLE(stdStream) _isatty(_fileno(stdStream))
-#elif defined(_POSIX_C_SOURCE) || defined(_XOPEN_SOURCE) || defined(_POSIX_SOURCE) || (defined(__APPLE__) && defined(__MACH__)) || \
-      defined(__DragonFly__) || defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__)  /* https://sourceforge.net/p/predef/wiki/OperatingSystems/ */
-#include  /* isatty */
-#define IS_CONSOLE(stdStream) isatty(fileno(stdStream))
-#else
-#define IS_CONSOLE(stdStream) 0
-#endif
 
 namespace pzstd {
 
diff --git a/contrib/pzstd/Pzstd.cpp b/contrib/pzstd/Pzstd.cpp
index 1eb4ce14c..6c580b3bc 100644
--- a/contrib/pzstd/Pzstd.cpp
+++ b/contrib/pzstd/Pzstd.cpp
@@ -6,6 +6,7 @@
  * LICENSE file in the root directory of this source tree) and the GPLv2 (found
  * in the COPYING file in the root directory of this source tree).
  */
+#include "platform.h"   /* Large Files support, SET_BINARY_MODE */
 #include "Pzstd.h"
 #include "SkippableFrame.h"
 #include "utils/FileSystem.h"
@@ -21,14 +22,6 @@
 #include 
 #include 
 
-#if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) || defined(__CYGWIN__)
-#  include     /* _O_BINARY */
-#  include        /* _setmode, _isatty */
-#  define SET_BINARY_MODE(file) { if (_setmode(_fileno(file), _O_BINARY) == -1) perror("Cannot set _O_BINARY"); }
-#else
-#  include    /* isatty */
-#  define SET_BINARY_MODE(file)
-#endif
 
 namespace pzstd {
 
diff --git a/zlibWrapper/gzguts.h b/zlibWrapper/gzguts.h
index 84651b88d..05bf4d9f4 100644
--- a/zlibWrapper/gzguts.h
+++ b/zlibWrapper/gzguts.h
@@ -1,5 +1,5 @@
 /* gzguts.h contains minimal changes required to be compiled with zlibWrapper:
- * - #include "zlib.h" was changed to #include "zstd_zlibwrapper.h"        
+ * - #include "zlib.h" was changed to #include "zstd_zlibwrapper.h"
  * - gz_statep was converted to union to work with -Wstrict-aliasing=1      */
 
 /* gzguts.h -- zlib internal header definitions for gz* operations
@@ -44,7 +44,7 @@
 #  include 
 #endif
 
-#if defined(_WIN32) || defined(__CYGWIN__)
+#if defined(_WIN32)
 #  define WIDECHAR
 #endif
 

From 71a521061756291e42e6d608d5c8010dccf3d428 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Fri, 21 Sep 2018 17:40:30 -0700
Subject: [PATCH 290/372] avoid recompiling dll every time under mingw

---
 lib/Makefile | 19 +++++++++++++------
 1 file changed, 13 insertions(+), 6 deletions(-)

diff --git a/lib/Makefile b/lib/Makefile
index cf8e45b0f..70fb649f7 100644
--- a/lib/Makefile
+++ b/lib/Makefile
@@ -94,8 +94,6 @@ else
 	SHARED_EXT_VER = $(SHARED_EXT).$(LIBVER)
 endif
 
-LIBZSTD = libzstd.$(SHARED_EXT_VER)
-
 
 .PHONY: default all clean install uninstall
 
@@ -111,19 +109,28 @@ libzstd.a: $(ZSTD_OBJ)
 libzstd.a-mt: CPPFLAGS += -DZSTD_MULTITHREAD
 libzstd.a-mt: libzstd.a
 
+ifneq (,$(filter Windows%,$(OS)))
+
+LIBZSTD = dll\libzstd.dll
+$(LIBZSTD): $(ZSTD_FILES)
+	@echo compiling dynamic library $(LIBVER)
+	@$(CC) $(FLAGS) -DZSTD_DLL_EXPORT=1 -shared $^ -o $@
+	dlltool -D $@ -d dll\libzstd.def -l dll\libzstd.lib
+
+else
+
+LIBZSTD = libzstd.$(SHARED_EXT_VER)
 $(LIBZSTD): LDFLAGS += -shared -fPIC -fvisibility=hidden
 $(LIBZSTD): $(ZSTD_FILES)
 	@echo compiling dynamic library $(LIBVER)
-ifneq (,$(filter Windows%,$(OS)))
-	@$(CC) $(FLAGS) -DZSTD_DLL_EXPORT=1 -shared $^ -o dll\libzstd.dll
-	dlltool -D dll\libzstd.dll -d dll\libzstd.def -l dll\libzstd.lib
-else
 	@$(CC) $(FLAGS) $^ $(LDFLAGS) $(SONAME_FLAGS) -o $@
 	@echo creating versioned links
 	@ln -sf $@ libzstd.$(SHARED_EXT_MAJOR)
 	@ln -sf $@ libzstd.$(SHARED_EXT)
+
 endif
 
+
 libzstd : $(LIBZSTD)
 
 libzstd-mt : CPPFLAGS += -DZSTD_MULTITHREAD

From 0e211bdd1825a4bf695c47cb7c62cad674625982 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Fri, 21 Sep 2018 18:23:32 -0700
Subject: [PATCH 291/372] fixed constant comparison on 32-bits systems

---
 tests/paramgrill.c | 20 ++++++++++++--------
 1 file changed, 12 insertions(+), 8 deletions(-)

diff --git a/tests/paramgrill.c b/tests/paramgrill.c
index dbddd242f..68e1fbc4d 100644
--- a/tests/paramgrill.c
+++ b/tests/paramgrill.c
@@ -1183,38 +1183,42 @@ static int createContexts(contexts_t* ctx, const char* dictFileName) {
     size_t readSize;
     ctx->cctx = ZSTD_createCCtx();
     ctx->dctx = ZSTD_createDCtx();
+    assert(ctx->cctx != NULL);
+    assert(ctx->dctx != NULL);
+
     if(dictFileName == NULL) {
         ctx->dictSize = 0;
         ctx->dictBuffer = NULL;
         return 0;
     }
-    ctx->dictSize = UTIL_getFileSize(dictFileName);
-    assert(ctx->dictSize != UTIL_FILESIZE_UNKNOWN);
+    {   U64 const dictFileSize = UTIL_getFileSize(dictFileName);
+        assert(dictFileSize != UTIL_FILESIZE_UNKNOWN);
+        ctx->dictSize = dictFileSize;
+        assert((U64)ctx->dictSize == dictFileSize); /* check overflow */
+    }
     ctx->dictBuffer = malloc(ctx->dictSize);
 
     f = fopen(dictFileName, "rb");
 
-    if(!f) {
+    if (f==NULL) {
         DISPLAY("unable to open file\n");
-        fclose(f);
         freeContexts(*ctx);
         return 1;
     }
 
-    if(ctx->dictSize > 64 MB || !(ctx->dictBuffer)) {
+    if (ctx->dictSize > 64 MB || !(ctx->dictBuffer)) {
         DISPLAY("dictionary too large\n");
         fclose(f);
         freeContexts(*ctx);
         return 1;
     }
     readSize = fread(ctx->dictBuffer, 1, ctx->dictSize, f);
-    if(readSize != ctx->dictSize) {
+    fclose(f);
+    if (readSize != ctx->dictSize) {
         DISPLAY("unable to read file\n");
-        fclose(f);
         freeContexts(*ctx);
         return 1;
     }
-    fclose(f);
     return 0;
 }
 

From 364041c6dd28b09e03ddd729d71f607d6936c5c6 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Sat, 22 Sep 2018 16:10:10 -0700
Subject: [PATCH 292/372] enforce minimum compression level limit

using ZSTD_minCLevel()
---
 programs/zstdcli.c | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/programs/zstdcli.c b/programs/zstdcli.c
index 3d4548a4b..6b2c5d10a 100644
--- a/programs/zstdcli.c
+++ b/programs/zstdcli.c
@@ -37,8 +37,8 @@
 #ifndef ZSTD_NODICT
 #  include "dibio.h"  /* ZDICT_cover_params_t, DiB_trainFromFiles() */
 #endif
-#define ZSTD_STATIC_LINKING_ONLY   /* ZSTD_maxCLevel */
-#include "zstd.h"     /* ZSTD_VERSION_STRING */
+#define ZSTD_STATIC_LINKING_ONLY   /* ZSTD_minCLevel */
+#include "zstd.h"     /* ZSTD_VERSION_STRING, ZSTD_maxCLevel */
 
 
 /*-************************************
@@ -634,9 +634,11 @@ int main(int argCount, const char* argv[])
                     if (longCommandWArg(&argument, "--fast")) {
                         /* Parse optional acceleration factor */
                         if (*argument == '=') {
+                            U32 const maxFast = (U32)-ZSTD_minCLevel();
                             U32 fastLevel;
                             ++argument;
                             fastLevel = readU32FromChar(&argument);
+                            if (fastLevel > maxFast) fastLevel = maxFast;
                             if (fastLevel) {
                               dictCLevel = cLevel = -(int)fastLevel;
                             } else {

From 0fc07eb1fd8bc7f661b0b16ad64f180234a6f443 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Sat, 22 Sep 2018 17:21:39 -0700
Subject: [PATCH 293/372] fixed zstd-decompress

which cannot support ZSTD_minCLevel()
---
 programs/zstdcli.c | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/programs/zstdcli.c b/programs/zstdcli.c
index 6b2c5d10a..c00d05258 100644
--- a/programs/zstdcli.c
+++ b/programs/zstdcli.c
@@ -631,6 +631,7 @@ int main(int argCount, const char* argv[])
                             compressionParams.windowLog = ldmWindowLog;
                         continue;
                     }
+#ifndef ZSTD_NOCOMPRESS   /* linking ZSTD_minCLevel() requires compression support */
                     if (longCommandWArg(&argument, "--fast")) {
                         /* Parse optional acceleration factor */
                         if (*argument == '=') {
@@ -652,6 +653,7 @@ int main(int argCount, const char* argv[])
                         }
                         continue;
                     }
+#endif
                     /* fall-through, will trigger bad_usage() later on */
                 }
 

From 292d8e4a831d7fa94be382824e51f3c14b5c7447 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Sun, 23 Sep 2018 23:57:30 -0700
Subject: [PATCH 294/372] added some tests based on limits.h

in order to ensure proper type mapping
when not using stdint.h
---
 lib/common/mem.h | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/lib/common/mem.h b/lib/common/mem.h
index 47d230017..2051bcad1 100644
--- a/lib/common/mem.h
+++ b/lib/common/mem.h
@@ -57,11 +57,23 @@ MEM_STATIC void MEM_check(void) { MEM_STATIC_ASSERT((sizeof(size_t)==4) || (size
   typedef  uint64_t U64;
   typedef   int64_t S64;
 #else
+# include 
+#if CHAR_BIT != 8
+#  error "this implementation requires char to be exactly 8-bit type"
+#endif
   typedef unsigned char      BYTE;
+#if USHRT_MAX != 65535
+#  error "this implementation requires short to be exactly 16-bit type"
+#endif
   typedef unsigned short      U16;
   typedef   signed short      S16;
+#if UINT_MAX != 4294967295
+#  error "this implementation requires int to be exactly 32-bit type"
+#endif
   typedef unsigned int        U32;
   typedef   signed int        S32;
+/* note : there are no limits defined for long long type in C90.
+ * limits exist in C99, however, in such case,  is preferred */
   typedef unsigned long long  U64;
   typedef   signed long long  S64;
 #endif

From 0250ac74ce261769ca550e93d11918915fd46633 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Mon, 24 Sep 2018 00:52:19 -0700
Subject: [PATCH 295/372] fixed minor scan-build warnings

---
 programs/fileio.c  | 1 +
 programs/zstdcli.c | 8 +++++---
 2 files changed, 6 insertions(+), 3 deletions(-)

diff --git a/programs/fileio.c b/programs/fileio.c
index e08920ba8..01944a3a0 100644
--- a/programs/fileio.c
+++ b/programs/fileio.c
@@ -1673,6 +1673,7 @@ static int FIO_decompressSrcFile(dRess_t ress, const char* dstFileName, const ch
 
     srcFile = FIO_openSrcFile(srcFileName);
     if (srcFile==NULL) return 1;
+    ress.srcBufferLoaded = 0;
 
     result = FIO_decompressFrames(ress, srcFile, dstFileName, srcFileName);
 
diff --git a/programs/zstdcli.c b/programs/zstdcli.c
index 3d4548a4b..24e05d83a 100644
--- a/programs/zstdcli.c
+++ b/programs/zstdcli.c
@@ -484,8 +484,6 @@ int main(int argCount, const char* argv[])
 
     /* init */
     (void)recursive; (void)cLevelLast;    /* not used when ZSTD_NOBENCH set */
-    (void)dictCLevel; (void)dictSelect; (void)dictID;  (void)maxDictSize; /* not used when ZSTD_NODICT set */
-    (void)ultra; (void)cLevel; (void)ldmFlag; /* not used when ZSTD_NOCOMPRESS set */
     (void)memLimit;   /* not used when ZSTD_NODECOMPRESS set */
     if (filenameTable==NULL) { DISPLAY("zstd: %s \n", strerror(errno)); exit(1); }
     filenameTable[0] = stdinmark;
@@ -956,6 +954,10 @@ int main(int argCount, const char* argv[])
             dictParams.zParams = zParams;
             operationResult = DiB_trainFromFiles(outFileName, maxDictSize, filenameTable, filenameIdx, blockSize, &dictParams, NULL, NULL, 0);
         }
+#else
+        (void)dictCLevel; (void)dictSelect; (void)dictID;  (void)maxDictSize; /* not used when ZSTD_NODICT set */
+        DISPLAYLEVEL(1, "training mode not available \n");
+        operationResult = 1;
 #endif
         goto _end;
     }
@@ -1014,7 +1016,7 @@ int main(int argCount, const char* argv[])
         else
           operationResult = FIO_compressMultipleFilenames(filenameTable, filenameIdx, outFileName, suffix, dictFileName, cLevel, &compressionParams);
 #else
-        (void)suffix;
+        (void)suffix; (void)ultra; (void)cLevel; (void)ldmFlag; /* not used when ZSTD_NOCOMPRESS set */
         DISPLAY("Compression not supported\n");
 #endif
     } else {  /* decompression or test */

From 63abaf217109421b97ab264600d77d5b0ee2787b Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Tue, 25 Sep 2018 15:57:28 -0700
Subject: [PATCH 296/372] fixed `!` tests

Sometimes, it's necessary to test that a certain command fail, as expected.
Such failure is actually a success, and must not stop the flow of tests.

Several tests were prefixed with `!` to invert return code.
This does not work : it effectively makes the tests pass no matter what.

Use instead function die(), which is meant to trap successes, and transform them into errors.
---
 tests/playTests.sh | 34 ++++++++++++++++++----------------
 1 file changed, 18 insertions(+), 16 deletions(-)

diff --git a/tests/playTests.sh b/tests/playTests.sh
index 19b502999..c643f6699 100755
--- a/tests/playTests.sh
+++ b/tests/playTests.sh
@@ -112,11 +112,13 @@ $ECHO "test : basic decompression"
 $ZSTD -df tmp.zst                 # trivial decompression case (overwrites tmp)
 $ECHO "test : too large compression level => auto-fix"
 $ZSTD -99 -f tmp  # too large compression level, automatic sized down
+$ZSTD -5000000000 -f tmp && die "too large numeric value : must fail"
 $ECHO "test : --fast aka negative compression levels"
 $ZSTD --fast -f tmp  # == -1
 $ZSTD --fast=3 -f tmp  # == -3
-$ZSTD --fast=200000 -f tmp  # == no compression
-! $ZSTD -c --fast=0 tmp > $INTOVOID # should fail
+$ZSTD --fast=200000 -f tmp  # too low compression level, automatic fixed
+$ZSTD --fast=5000000000 -f tmp && die "too large numeric value : must fail"
+$ZSTD -c --fast=0 tmp > $INTOVOID && die "--fast must not accept value 0"
 $ECHO "test : too large numeric argument"
 $ZSTD --fast=9999999999 -f tmp  && die "should have refused numeric value"
 $ECHO "test : compress to stdout"
@@ -798,32 +800,32 @@ $ZSTD -l *.zst
 $ZSTD -lv *.zst
 
 $ECHO "\n===>  zstd --list/-l error detection tests "
-! $ZSTD -l tmp1 tmp1.zst
-! $ZSTD --list tmp*
-! $ZSTD -lv tmp1*
-! $ZSTD --list -v tmp2 tmp12.zst
+$ZSTD -l tmp1 tmp1.zst && die "-l must fail on non-zstd file"
+$ZSTD --list tmp* && die "-l must fail on non-zstd file"
+$ZSTD -lv tmp1* && die "-l must fail on non-zstd file"
+$ZSTD --list -v tmp2 tmp12.zst && die "-l must fail on non-zstd file"
 
 $ECHO "\n===>  zstd --list/-l errors when presented with stdin / no files"
-! $ZSTD -l
-! $ZSTD -l -
-! $ZSTD -l < tmp1.zst
-! $ZSTD -l - < tmp1.zst
-! $ZSTD -l - tmp1.zst
-! $ZSTD -l - tmp1.zst < tmp1.zst
-$ZSTD -l tmp1.zst < tmp1.zst # but doesn't error just because stdin is not a tty
+$ZSTD -l && die "-l must fail on empty list of files"
+$ZSTD -l - && die "-l does not work on stdin"
+$ZSTD -l < tmp1.zst && die "-l does not work on stdin"
+$ZSTD -l - < tmp1.zst && die "-l does not work on stdin"
+$ZSTD -l - tmp1.zst && die "-l does not work on stdin"
+$ZSTD -l - tmp1.zst < tmp1.zst && die "-l does not work on stdin"
+$ZSTD -l tmp1.zst < tmp2.zst # this will check tmp1.zst, but not tmp2.zst, which is not an error : zstd simply doesn't read stdin in this case. It must not error just because stdin is not a tty
 
 $ECHO "\n===>  zstd --list/-l test with null files "
 ./datagen -g0 > tmp5
 $ZSTD tmp5
 $ZSTD -l tmp5.zst
-! $ZSTD -l tmp5*
+$ZSTD -l tmp5* && die "-l must fail on non-zstd file"
 $ZSTD -lv tmp5.zst | grep "Decompressed Size: 0.00 KB (0 B)"  # check that 0 size is present in header
-! $ZSTD -lv tmp5*
+$ZSTD -lv tmp5* && die "-l must fail on non-zstd file"
 
 $ECHO "\n===>  zstd --list/-l test with no content size field "
 ./datagen -g513K | $ZSTD > tmp6.zst
 $ZSTD -l tmp6.zst
-! $ZSTD -lv tmp6.zst | grep "Decompressed Size:"  # must NOT be present in header
+$ZSTD -lv tmp6.zst | grep "Decompressed Size:"  && die "Field :Decompressed Size: should not be available in this compressed file"
 
 $ECHO "\n===>   zstd --list/-l test with no checksum "
 $ZSTD -f --no-check tmp1

From 6c51bf420c51246ec51a55f801f4853eaf167d8a Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Mon, 24 Sep 2018 18:16:08 -0700
Subject: [PATCH 297/372] bounds for --adapt mode

can supply min and max compression level through advanced command :
--adapt=min=#,max=#
---
 programs/fileio.c  | 18 +++++++++++++++++-
 programs/fileio.h  | 31 ++++++++++++++++++++-----------
 programs/zstd.1.md |  3 ++-
 programs/zstdcli.c | 42 +++++++++++++++++++++++++++++++++++++++++-
 tests/playTests.sh | 14 ++++++++++----
 5 files changed, 90 insertions(+), 18 deletions(-)

diff --git a/programs/fileio.c b/programs/fileio.c
index e4ad6c4f1..53f72aa72 100644
--- a/programs/fileio.c
+++ b/programs/fileio.c
@@ -292,6 +292,20 @@ void FIO_setAdaptiveMode(unsigned adapt) {
         EXM_THROW(1, "Adaptive mode is not compatible with single thread mode \n");
     g_adaptiveMode = adapt;
 }
+static int g_minAdaptLevel = -50;   /* initializing this value requires a constant, so ZSTD_minCLevel() doesn't work */
+void FIO_setAdaptMin(int minCLevel)
+{
+#ifndef ZSTD_NOCOMPRESS
+    assert(minCLevel >= ZSTD_minCLevel());
+#endif
+    g_minAdaptLevel = minCLevel;
+}
+static int g_maxAdaptLevel = 22;   /* initializing this value requires a constant, so ZSTD_maxCLevel() doesn't work */
+void FIO_setAdaptMax(int maxCLevel)
+{
+    g_maxAdaptLevel = maxCLevel;
+}
+
 static U32 g_ldmFlag = 0;
 void FIO_setLdmFlag(unsigned ldmFlag) {
     g_ldmFlag = (ldmFlag>0);
@@ -954,13 +968,15 @@ FIO_compressZstdFrame(const cRess_t* ressPtr,
                         if (speedChange == slower) {
                             DISPLAYLEVEL(6, "slower speed , higher compression \n")
                             compressionLevel ++;
-                            compressionLevel += (compressionLevel == 0);   /* skip 0 */
                             if (compressionLevel > ZSTD_maxCLevel()) compressionLevel = ZSTD_maxCLevel();
+                            if (compressionLevel > g_maxAdaptLevel) compressionLevel = g_maxAdaptLevel;
+                            compressionLevel += (compressionLevel == 0);   /* skip 0 */
                             ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_compressionLevel, (unsigned)compressionLevel);
                         }
                         if (speedChange == faster) {
                             DISPLAYLEVEL(6, "faster speed , lighter compression \n")
                             compressionLevel --;
+                            if (compressionLevel < g_minAdaptLevel) compressionLevel = g_minAdaptLevel;
                             compressionLevel -= (compressionLevel == 0);   /* skip 0 */
                             ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_compressionLevel, (unsigned)compressionLevel);
                         }
diff --git a/programs/fileio.h b/programs/fileio.h
index 94789929e..4c7049cb7 100644
--- a/programs/fileio.h
+++ b/programs/fileio.h
@@ -48,21 +48,23 @@ typedef enum { FIO_zstdCompression, FIO_gzipCompression, FIO_xzCompression, FIO_
 ***************************************/
 void FIO_setCompressionType(FIO_compressionType_t compressionType);
 void FIO_overwriteMode(void);
-void FIO_setNotificationLevel(unsigned level);
-void FIO_setSparseWrite(unsigned sparse);  /**< 0: no sparse; 1: disable on stdout; 2: always enabled */
-void FIO_setDictIDFlag(unsigned dictIDFlag);
-void FIO_setChecksumFlag(unsigned checksumFlag);
-void FIO_setRemoveSrcFile(unsigned flag);
-void FIO_setMemLimit(unsigned memLimit);
-void FIO_setNbWorkers(unsigned nbWorkers);
-void FIO_setBlockSize(unsigned blockSize);
-void FIO_setOverlapLog(unsigned overlapLog);
 void FIO_setAdaptiveMode(unsigned adapt);
+void FIO_setAdaptMin(int minCLevel);
+void FIO_setAdaptMax(int maxCLevel);
+void FIO_setBlockSize(unsigned blockSize);
+void FIO_setChecksumFlag(unsigned checksumFlag);
+void FIO_setDictIDFlag(unsigned dictIDFlag);
+void FIO_setLdmBucketSizeLog(unsigned ldmBucketSizeLog);
 void FIO_setLdmFlag(unsigned ldmFlag);
+void FIO_setLdmHashEveryLog(unsigned ldmHashEveryLog);
 void FIO_setLdmHashLog(unsigned ldmHashLog);
 void FIO_setLdmMinMatch(unsigned ldmMinMatch);
-void FIO_setLdmBucketSizeLog(unsigned ldmBucketSizeLog);
-void FIO_setLdmHashEveryLog(unsigned ldmHashEveryLog);
+void FIO_setMemLimit(unsigned memLimit);
+void FIO_setNbWorkers(unsigned nbWorkers);
+void FIO_setNotificationLevel(unsigned level);
+void FIO_setOverlapLog(unsigned overlapLog);
+void FIO_setRemoveSrcFile(unsigned flag);
+void FIO_setSparseWrite(unsigned sparse);  /**< 0: no sparse; 1: disable on stdout; 2: always enabled */
 
 
 /*-*************************************
@@ -79,6 +81,7 @@ int FIO_decompressFilename (const char* outfilename, const char* infilename, con
 
 int FIO_listMultipleFiles(unsigned numFiles, const char** filenameTable, int displayLevel);
 
+
 /*-*************************************
 *  Multiple File functions
 ***************************************/
@@ -96,9 +99,15 @@ int FIO_decompressMultipleFilenames(const char** srcNamesTable, unsigned nbFiles
                                     const char* dictFileName);
 
 
+/*-*************************************
+*  Advanced stuff (should actually be hosted elsewhere)
+***************************************/
+
 /* custom crash signal handler */
 void FIO_addAbortHandler(void);
 
+
+
 #if defined (__cplusplus)
 }
 #endif
diff --git a/programs/zstd.1.md b/programs/zstd.1.md
index fcc1944bf..c0c04698d 100644
--- a/programs/zstd.1.md
+++ b/programs/zstd.1.md
@@ -134,9 +134,10 @@ the last one takes effect.
     This mode is the only one available when multithread support is disabled.
     Single-thread mode features lower memory usage.
     Final compressed result is slightly different from `-T1`.
-* `--adapt` :
+* `--adapt[=min=#,max=#]` :
     `zstd` will dynamically adapt compression level to perceived I/O conditions.
     Compression level adaptation can be observed live by using command `-v`.
+    Adaptation can be constrained between supplied `min` and `max` levels.
     The feature works when combined with multi-threading and `--long` mode.
     It does not work with `--single-thread`.
     It sets window size to 8 MB by default (can be changed manually, see `wlog`).
diff --git a/programs/zstdcli.c b/programs/zstdcli.c
index 9ace49c4e..1545d1cac 100644
--- a/programs/zstdcli.c
+++ b/programs/zstdcli.c
@@ -138,8 +138,8 @@ static int usage_advanced(const char* programName)
 #ifndef ZSTD_NOCOMPRESS
     DISPLAY( "--ultra : enable levels beyond %i, up to %i (requires more memory)\n", ZSTDCLI_CLEVEL_MAX, ZSTD_maxCLevel());
     DISPLAY( "--long[=#]: enable long distance matching with given window log (default: %u)\n", g_defaultMaxWindowLog);
-    DISPLAY( "--adapt : automatically adapt compression level to I/O conditions \n");
     DISPLAY( "--fast[=#]: switch to ultra fast compression level (default: %u)\n", 1);
+    DISPLAY( "--adapt : dynamically adapt compression level to I/O conditions \n");
 #ifdef ZSTD_MULTITHREAD
     DISPLAY( " -T#    : spawns # compression threads (default: 1, 0==# cores) \n");
     DISPLAY( " -B#    : select size of each job (default: 0==automatic) \n");
@@ -366,6 +366,30 @@ static ZDICT_fastCover_params_t defaultFastCoverParams(void)
 #endif
 
 
+/** parseAdaptParameters() :
+ *  reads adapt parameters from *stringPtr (e.g. "--zstd=min=1,max=19) and store them into adaptMinPtr and adaptMaxPtr.
+ *  Both adaptMinPtr and adaptMaxPtr must be already allocated and correctly initialized.
+ *  There is no guarantee that any of these values will be updated.
+ *  @return 1 means that parsing was successful,
+ *  @return 0 in case of malformed parameters
+ */
+static unsigned parseAdaptParameters(const char* stringPtr, int* adaptMinPtr, int* adaptMaxPtr)
+{
+    for ( ; ;) {
+        if (longCommandWArg(&stringPtr, "min=")) { *adaptMinPtr = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
+        if (longCommandWArg(&stringPtr, "max=")) { *adaptMaxPtr = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
+        DISPLAYLEVEL(4, "invalid compression parameter \n");
+        return 0;
+    }
+    if (stringPtr[0] != 0) return 0; /* check the end of string */
+    if (*adaptMinPtr > *adaptMaxPtr) {
+        DISPLAYLEVEL(4, "incoherent adaptation limits \n");
+        return 0;
+    }
+    return 1;
+}
+
+
 /** parseCompressionParameters() :
  *  reads compression parameters from *stringPtr (e.g. "--zstd=wlog=23,clog=23,hlog=22,slog=6,slen=3,tlen=48,strat=6") into *params
  *  @return 1 means that compression parameters were correct
@@ -430,6 +454,15 @@ typedef enum { zom_compress, zom_decompress, zom_test, zom_bench, zom_train, zom
 
 #define CLEAN_RETURN(i) { operationResult = (i); goto _end; }
 
+#ifdef ZSTD_NOCOMPRESS
+/* symbols from compression library are not defined and should not be invoked */
+# define MINCLEVEL  -50
+# define MAXCLEVEL   22
+#else
+# define MINCLEVEL  ZSTD_minCLevel()
+# define MAXCLEVEL  ZSTD_maxCLevel()
+#endif
+
 int main(int argCount, const char* argv[])
 {
     int argNb,
@@ -440,6 +473,8 @@ int main(int argCount, const char* argv[])
         main_pause = 0,
         nbWorkers = 0,
         adapt = 0,
+        adaptMin = MINCLEVEL,
+        adaptMax = MAXCLEVEL,
         nextArgumentIsOutFileName = 0,
         nextArgumentIsMaxDict = 0,
         nextArgumentIsDictID = 0,
@@ -559,6 +594,7 @@ int main(int argCount, const char* argv[])
                     if (!strcmp(argument, "--rm")) { FIO_setRemoveSrcFile(1); continue; }
                     if (!strcmp(argument, "--priority=rt")) { setRealTimePrio = 1; continue; }
                     if (!strcmp(argument, "--adapt")) { adapt = 1; continue; }
+                    if (longCommandWArg(&argument, "--adapt=")) { adapt = 1; if (!parseAdaptParameters(argument, &adaptMin, &adaptMax)) CLEAN_RETURN(badusage(programName)); continue; }
                     if (!strcmp(argument, "--single-thread")) { nbWorkers = 0; singleThread = 1; continue; }
                     if (!strcmp(argument, "--format=zstd")) { suffix = ZSTD_EXTENSION; FIO_setCompressionType(FIO_zstdCompression); continue; }
 #ifdef ZSTD_GZCOMPRESS
@@ -1014,6 +1050,10 @@ int main(int argCount, const char* argv[])
         if (g_ldmBucketSizeLog != LDM_PARAM_DEFAULT) FIO_setLdmBucketSizeLog(g_ldmBucketSizeLog);
         if (g_ldmHashEveryLog != LDM_PARAM_DEFAULT) FIO_setLdmHashEveryLog(g_ldmHashEveryLog);
         FIO_setAdaptiveMode(adapt);
+        FIO_setAdaptMin(adaptMin);
+        FIO_setAdaptMax(adaptMax);
+        if (adaptMin > cLevel) cLevel = adaptMin;
+        if (adaptMax < cLevel) cLevel = adaptMax;
 
         if ((filenameIdx==1) && outFileName)
           operationResult = FIO_compressFilename(outFileName, filenameTable[0], dictFileName, cLevel, compressionParams);
diff --git a/tests/playTests.sh b/tests/playTests.sh
index 5898d183f..e49b23444 100755
--- a/tests/playTests.sh
+++ b/tests/playTests.sh
@@ -103,6 +103,7 @@ else
 fi
 
 
+
 $ECHO "\n===>  simple tests "
 
 ./datagen > tmp
@@ -811,11 +812,20 @@ roundTripTest -g1M -P50 "1 --single-thread --long=29" " --long=28 --memory=512MB
 roundTripTest -g1M -P50 "1 --single-thread --long=29" " --zstd=wlog=28 --memory=512MB"
 
 
+$ECHO "\n===>   adaptive mode "
+roundTripTest -g270000000 " --adapt"
+roundTripTest -g27000000 " --adapt=min=1,max=4"
+./datagen > tmp
+$ZSTD -f -vv --adapt=min=10,max=9 tmp && die "--adapt must fail on incoherent bounds"
+
+
 if [ "$1" != "--test-large-data" ]; then
     $ECHO "Skipping large data tests"
     exit 0
 fi
 
+
+
 $ECHO "\n===>   large files tests "
 
 roundTripTest -g270000000 1
@@ -858,10 +868,6 @@ roundTripTest -g700M -P50 "1 --single-thread --long=29"
 roundTripTest -g600M -P50 "1 --single-thread --long --zstd=wlog=29,clog=28"
 
 
-$ECHO "\n===>   adaptive mode "
-roundTripTest -g270000000 " --adapt"
-
-
 if [ -n "$hasMT" ]
 then
     $ECHO "\n===>   zstdmt long round-trip tests "

From 72a3adf826a79847e56f02e693da205c98d3f579 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Tue, 25 Sep 2018 16:34:26 -0700
Subject: [PATCH 298/372] updated format documentation

to match last edits of RFC8478.
---
 doc/zstd_compression_format.md | 27 ++++++++++++++-------------
 1 file changed, 14 insertions(+), 13 deletions(-)

diff --git a/doc/zstd_compression_format.md b/doc/zstd_compression_format.md
index c57b58269..e562e628b 100644
--- a/doc/zstd_compression_format.md
+++ b/doc/zstd_compression_format.md
@@ -16,7 +16,7 @@ Distribution of this document is unlimited.
 
 ### Version
 
-0.2.9 (05/09/18)
+0.3.0 (25/09/18)
 
 
 Introduction
@@ -72,7 +72,7 @@ A frame is completely independent, has a defined beginning and end,
 and a set of parameters which tells the decoder how to decompress it.
 
 A frame encapsulates one or multiple __blocks__.
-Each block can be compressed or not,
+Each block contains arbitrary content, which is described by its header,
 and has a guaranteed maximum content size, which depends on frame parameters.
 Unlike frames, each block depends on previous blocks for proper decoding.
 However, each block can be decompressed without waiting for its successor,
@@ -591,7 +591,7 @@ It is the number of bytes to be copied (or extracted) from the Literals Section.
 A match copy command specifies an offset and a length.
 
 When all _sequences_ are decoded,
-if there are literals left in the _literal section_,
+if there are literals left in the _literals section_,
 these bytes are added at the end of the block.
 
 This is described in more detail in [Sequence Execution](#sequence-execution).
@@ -608,7 +608,7 @@ followed by the bitstream.
 | -------------------------- | ------------------------- | ---------------- | ---------------------- | --------- |
 
 To decode the `Sequences_Section`, it's required to know its size.
-This size is deduced from the literals section size:
+Its size is deduced from the size of `Literals_Section`:
 `Sequences_Section_Size = Block_Size - Literals_Section_Size`.
 
 
@@ -805,7 +805,7 @@ one and ending with the first.
 
 ##### Decoding a sequence
 For each of the symbol types, the FSE state can be used to determine the appropriate code.
-The code then defines the baseline and number of bits to read for each type.
+The code then defines the `Baseline` and `Number_of_Bits` to read for each type.
 See the [description of the codes] for how to determine these values.
 
 [description of the codes]: #the-codes-for-literals-lengths-match-lengths-and-offsets
@@ -872,8 +872,8 @@ they are combined to produce the decoded content of a block.
 
 Each sequence consists of a tuple of (`literals_length`, `offset_value`, `match_length`),
 decoded as described in the [Sequences Section](#sequences-section).
-To execute a sequence, first copy `literals_length` bytes from the literals section
-to the output.
+To execute a sequence, first copy `literals_length` bytes
+from the decoded literals to the output.
 
 Then `match_length` bytes are copied from previous decoded data.
 The offset to copy from is determined by `offset_value`:
@@ -1219,8 +1219,8 @@ It gives the following series of weights :
 
 The decoder will do the inverse operation :
 having collected weights of literal symbols from `0` to `4`,
-it knows the last literal, `5`, is present with a non-zero weight.
-The weight of `5` can be determined by advancing to the next power of 2.
+it knows the last literal, `5`, is present with a non-zero `Weight`.
+The `Weight` of `5` can be determined by advancing to the next power of 2.
 The sum of `2^(Weight-1)` (excluding 0's) is :
 `8 + 4 + 2 + 0 + 1 = 15`.
 Nearest larger power of 2 value is 16.
@@ -1265,7 +1265,7 @@ To decode an FSE bitstream, it is necessary to know its compressed size.
 Compressed size is provided by `headerByte`.
 It's also necessary to know its _maximum possible_ decompressed size,
 which is `255`, since literal values span from `0` to `255`,
-and last symbol's weight is not represented.
+and last symbol's `Weight` is not represented.
 
 An FSE bitstream starts by a header, describing probabilities distribution.
 It will create a Decoding Table.
@@ -1275,7 +1275,7 @@ For more description see the [FSE header description](#fse-table-description)
 The Huffman header compression uses 2 states,
 which share the same FSE distribution table.
 The first state (`State1`) encodes the even indexed symbols,
-and the second (`State2`) encodes the odd indexes.
+and the second (`State2`) encodes the odd indexed symbols.
 `State1` is initialized first, and then `State2`, and they take turns
 decoding a single symbol and updating their state.
 For more details on these FSE operations, see the [FSE section](#fse).
@@ -1296,7 +1296,7 @@ Number_of_Bits = (Weight>0) ? Max_Number_of_Bits + 1 - Weight : 0
 Symbols are sorted by `Weight`.
 Within same `Weight`, symbols keep natural sequential order.
 Symbols with a `Weight` of zero are removed.
-Then, starting from lowest weight, prefix codes are distributed in sequential order.
+Then, starting from lowest `Weight`, prefix codes are distributed in sequential order.
 
 __Example__ :
 Let's presume the following list of weights has been decoded :
@@ -1323,7 +1323,7 @@ Each bitstream must be read _backward_,
 that is starting from the end down to the beginning.
 Therefore it's necessary to know the size of each bitstream.
 
-It's also necessary to know exactly which _bit_ is the latest.
+It's also necessary to know exactly which _bit_ is the last one.
 This is detected by a final bit flag :
 the highest bit of latest byte is a final-bit-flag.
 Consequently, a last byte of `0` is not possible.
@@ -1629,6 +1629,7 @@ or at least provide a meaningful error code explaining for which reason it canno
 
 Version changes
 ---------------
+- 0.3.0 : minor edits to match RFC8478
 - 0.2.9 : clarifications for huffman weights direct representation, by Ulrich Kunitz
 - 0.2.8 : clarifications for IETF RFC discuss
 - 0.2.7 : clarifications from IETF RFC review, by Vijay Gurbani and Nick Terrell

From 08f68d83c5cdcc34a0d99beab4d7d70018c75f79 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Tue, 25 Sep 2018 16:56:53 -0700
Subject: [PATCH 299/372] fixed usage of grep in Makefile

when terminal uses colors
as suggested by @danielshir (#1294)
---
 Makefile          |  6 ++++--
 lib/Makefile      |  4 +++-
 programs/Makefile | 28 ++++++++++++++++++++++++----
 3 files changed, 31 insertions(+), 7 deletions(-)

diff --git a/Makefile b/Makefile
index a17900450..4011250c5 100644
--- a/Makefile
+++ b/Makefile
@@ -118,6 +118,8 @@ ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD Dr
 HOST_OS = POSIX
 CMAKE_PARAMS = -DZSTD_BUILD_CONTRIB:BOOL=ON -DZSTD_BUILD_STATIC:BOOL=ON -DZSTD_BUILD_TESTS:BOOL=ON -DZSTD_ZLIB_SUPPORT:BOOL=ON -DZSTD_LZMA_SUPPORT:BOOL=ON -DCMAKE_BUILD_TYPE=Release
 
+EGREP = egrep --color=never
+
 # Print a two column output of targets and their description. To add a target description, put a
 # comment in the Makefile with the format "## : ".  For example:
 #
@@ -126,12 +128,12 @@ CMAKE_PARAMS = -DZSTD_BUILD_CONTRIB:BOOL=ON -DZSTD_BUILD_STATIC:BOOL=ON -DZSTD_B
 list:
 	@TARGETS=$$($(MAKE) -pRrq -f $(lastword $(MAKEFILE_LIST)) : 2>/dev/null \
 		| awk -v RS= -F: '/^# File/,/^# Finished Make data base/ {if ($$1 !~ "^[#.]") {print $$1}}' \
-		| egrep -v  -e '^[^[:alnum:]]' | sort); \
+		| $(EGREP) -v  -e '^[^[:alnum:]]' | sort); \
 	{ \
 	    printf "Target Name\tDescription\n"; \
 	    printf "%0.s-" {1..16}; printf "\t"; printf "%0.s-" {1..40}; printf "\n"; \
 	    for target in $$TARGETS; do \
-	        line=$$(egrep "^##[[:space:]]+$$target:" $(lastword $(MAKEFILE_LIST))); \
+	        line=$$($(EGREP) "^##[[:space:]]+$$target:" $(lastword $(MAKEFILE_LIST))); \
 	        description=$$(echo $$line | awk '{i=index($$0,":"); print substr($$0,i+1)}' | xargs); \
 	        printf "$$target\t$$description\n"; \
 	    done \
diff --git a/lib/Makefile b/lib/Makefile
index 70fb649f7..c102d71b9 100644
--- a/lib/Makefile
+++ b/lib/Makefile
@@ -31,6 +31,8 @@ DEBUGFLAGS= -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \
 CFLAGS  += $(DEBUGFLAGS) $(MOREFLAGS)
 FLAGS    = $(CPPFLAGS) $(CFLAGS)
 
+GREP = grep --color=never
+
 ZSTDCOMMON_FILES := $(sort $(wildcard common/*.c))
 ZSTDCOMP_FILES := $(sort $(wildcard compress/*.c))
 ZSTDDECOMP_FILES := $(sort $(wildcard decompress/*.c))
@@ -72,7 +74,7 @@ endif
 
 ifneq ($(ZSTD_LEGACY_SUPPORT), 0)
 ifeq ($(shell test $(ZSTD_LEGACY_SUPPORT) -lt 8; echo $$?), 0)
-	ZSTD_FILES += $(shell ls legacy/*.c | grep 'v0[$(ZSTD_LEGACY_SUPPORT)-7]')
+	ZSTD_FILES += $(shell ls legacy/*.c | $(GREP) 'v0[$(ZSTD_LEGACY_SUPPORT)-7]')
 endif
 	CPPFLAGS += -I./legacy
 endif
diff --git a/programs/Makefile b/programs/Makefile
index f1a96325c..2a02bec55 100644
--- a/programs/Makefile
+++ b/programs/Makefile
@@ -27,9 +27,11 @@ LIBVER_MINOR := $(shell echo $(LIBVER_MINOR_SCRIPT))
 LIBVER_PATCH := $(shell echo $(LIBVER_PATCH_SCRIPT))
 LIBVER  := $(shell echo $(LIBVER_SCRIPT))
 
-ZSTD_VERSION=$(LIBVER)
+ZSTD_VERSION = $(LIBVER)
 
-ifeq ($(shell $(CC) -v 2>&1 | grep -c "gcc version "), 1)
+GREP = grep --color=never
+
+ifeq ($(shell $(CC) -v 2>&1 | $(GREP) -c "gcc version "), 1)
 ALIGN_LOOP = -falign-loops=32
 else
 ALIGN_LOOP =
@@ -62,7 +64,7 @@ ZSTD_LEGACY_SUPPORT ?= 4
 ZSTDLEGACY_FILES :=
 ifneq ($(ZSTD_LEGACY_SUPPORT), 0)
 ifeq ($(shell test $(ZSTD_LEGACY_SUPPORT) -lt 8; echo $$?), 0)
-	ZSTDLEGACY_FILES += $(shell ls $(ZSTDDIR)/legacy/*.c | grep 'v0[$(ZSTD_LEGACY_SUPPORT)-7]')
+	ZSTDLEGACY_FILES += $(shell ls $(ZSTDDIR)/legacy/*.c | $(GREP) 'v0[$(ZSTD_LEGACY_SUPPORT)-7]')
 endif
 	CPPFLAGS += -I$(ZSTDDIR)/legacy
 else
@@ -260,9 +262,27 @@ preview-man: clean-man man
 #-----------------------------------------------------------------------------
 ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS))
 
+EGREP = egrep --color=never
+
+# Print a two column output of targets and their description. To add a target description, put a
+# comment in the Makefile with the format "## : ".  For example:
+#
+## list: Print all targets and their descriptions (if provided)
 .PHONY: list
 list:
-	@$(MAKE) -pRrq -f $(lastword $(MAKEFILE_LIST)) : 2>/dev/null | awk -v RS= -F: '/^# File/,/^# Finished Make data base/ {if ($$1 !~ "^[#.]") {print $$1}}' | sort | egrep -v -e '^[^[:alnum:]]' -e '^$@$$' | xargs
+	@TARGETS=$$($(MAKE) -pRrq -f $(lastword $(MAKEFILE_LIST)) : 2>/dev/null \
+		| awk -v RS= -F: '/^# File/,/^# Finished Make data base/ {if ($$1 !~ "^[#.]") {print $$1}}' \
+		| $(EGREP) -v  -e '^[^[:alnum:]]' | sort); \
+	{ \
+	    printf "Target Name\tDescription\n"; \
+	    printf "%0.s-" {1..16}; printf "\t"; printf "%0.s-" {1..40}; printf "\n"; \
+	    for target in $$TARGETS; do \
+	        line=$$($(EGREP) "^##[[:space:]]+$$target:" $(lastword $(MAKEFILE_LIST))); \
+	        description=$$(echo $$line | awk '{i=index($$0,":"); print substr($$0,i+1)}' | xargs); \
+	        printf "$$target\t$$description\n"; \
+	    done \
+	} | column -t -s $$'\t'
+
 
 DESTDIR     ?=
 # directory variables : GNU conventions prefer lowercase

From 3ff604084842f6a7624ae1ab2c2d643594ba4adf Mon Sep 17 00:00:00 2001
From: Nick Terrell 
Date: Tue, 25 Sep 2018 16:10:17 -0700
Subject: [PATCH 300/372] Publish artifacts with CircleCI

* Updates CircleCI to use workflows.
  We can now specify any number of test jobs to run in parallel.
* Switch the image to `buildpack-deps:trusty` which is only 500 MB
  instead of 7 GB, so that saves 7 minutes to download it if it isn't
  already cached on the host.
* Publish the source tarball and sha256sum as artifacts.
* If the `GITHUB_TOKEN` environment variable is set, we will also
  add the tarball + sha256sum to the tagged release, after manual
  approval.
---
 .circleci/config.yml | 203 ++++++++++++++++++++++---------------------
 tests/fuzz/fuzz.py   |  11 +--
 2 files changed, 109 insertions(+), 105 deletions(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 7af8b41ed..b0138d4ed 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -1,108 +1,111 @@
-# This configuration was automatically generated from a CircleCI 1.0 config.
-# It should include any build commands you had along with commands that CircleCI
-# inferred from your project structure. We strongly recommend you read all the
-# comments in this file to understand the structure of CircleCI 2.0, as the idiom
-# for configuration has changed substantially in 2.0 to allow arbitrary jobs rather
-# than the prescribed lifecycle of 1.0. In general, we recommend using this generated
-# configuration as a reference rather than using it in production, though in most
-# cases it should duplicate the execution of your original 1.0 config.
 version: 2
+
+references:
+  # Install the dependencies required for tests.
+  # Add the step "- *install-dependencies" to the beginning of your job to run
+  # this command.
+  install-dependencies: &install-dependencies
+    run:
+      name: Install dependencies
+      # TODO: We can split these dependencies up by job to reduce installation
+      # time.
+      command: |
+        sudo dpkg --add-architecture i386
+        sudo apt-get -y -qq update
+        sudo apt-get -y install \
+            gcc-multilib-powerpc-linux-gnu gcc-arm-linux-gnueabi \
+            libc6-dev-armel-cross gcc-aarch64-linux-gnu libc6-dev-arm64-cross \
+            libc6-dev-ppc64-powerpc-cross
+
 jobs:
-  build:
-    working_directory: ~/facebook/zstd
-    parallelism: 1
-    shell: /bin/bash --login
-    # CircleCI 2.0 does not support environment variables that refer to each other the same way as 1.0 did.
-    # If any of these refer to each other, rewrite them so that they don't or see https://circleci.com/docs/2.0/env-vars/#interpolating-environment-variables-to-set-other-environment-variables .
+  # the first half of the jobs are in this test
+  short-tests-0:
+    # TODO: Create a small custom docker image with all the dependencies we need
+    #       preinstalled to reduce installation time.
+    docker:
+      - image: circleci/buildpack-deps:bionic
+    steps:
+      - checkout
+      - *install-dependencies
+      - run:
+          name: Test
+          command: |
+            cc -v; CFLAGS="-O0 -Werror" make all && make clean
+            make c99build         ; make clean
+            make c11build         ; make clean
+            make aarch64build     ; make clean
+            make -j regressiontest; make clean
+            make shortest         ; make clean
+            make cxxtest          ; make clean
+  # the second half of the jobs are in this test
+  short-tests-1:
+    docker:
+      - image: circleci/buildpack-deps:bionic
+    steps:
+      - checkout
+      - *install-dependencies
+      - run:
+          name: Test
+          command: |
+            make gnu90build; make clean
+            make gnu99build; make clean
+            make ppc64build; make clean
+            make ppcbuild  ; make clean
+            make armbuild  ; make clean
+            make -C tests test-legacy test-longmatch test-symbols; make clean
+            make -C lib libzstd-nomt; make clean
+  # This step is only run on release tags.
+  # It publishes the source tarball as artifacts and if the GITHUB_TOKEN
+  # environment variable is set it will publish the source tarball to the
+  # tagged release.
+  publish-github-release:
+    docker:
+      - image: cibuilds/github:0.12.0
     environment:
       CIRCLE_ARTIFACTS: /tmp/circleci-artifacts
-      CIRCLE_TEST_REPORTS: /tmp/circleci-test-results
-    # In CircleCI 1.0 we used a pre-configured image with a large number of languages and other packages.
-    # In CircleCI 2.0 you can now specify your own image, or use one of our pre-configured images.
-    # The following configuration line tells CircleCI to use the specified docker image as the runtime environment for you job.
-    # We have selected a pre-built image that mirrors the build environment we use on
-    # the 1.0 platform, but we recommend you choose an image more tailored to the needs
-    # of each job. For more information on choosing an image (or alternatively using a
-    # VM instead of a container) see https://circleci.com/docs/2.0/executor-types/
-    # To see the list of pre-built images that CircleCI provides for most common languages see
-    # https://circleci.com/docs/2.0/circleci-images/
-    docker:
-    - image: circleci/build-image:ubuntu-14.04-XXL-upstart-1189-5614f37
-      command: /sbin/init
     steps:
-    # Machine Setup
-    #   If you break your build into multiple jobs with workflows, you will probably want to do the parts of this that are relevant in each
-    # The following `checkout` command checks out your code to your working directory. In 1.0 we did this implicitly. In 2.0 you can choose where in the course of a job your code should be checked out.
-    - checkout
-    # Prepare for artifact and test results  collection equivalent to how it was done on 1.0.
-    # In many cases you can simplify this from what is generated here.
-    # 'See docs on artifact collection here https://circleci.com/docs/2.0/artifacts/'
-    - run: mkdir -p $CIRCLE_ARTIFACTS $CIRCLE_TEST_REPORTS
-    # Dependencies
-    #   This would typically go in either a build or a build-and-test job when using workflows
-    # Restore the dependency cache
-    - restore_cache:
-        keys:
-        # This branch if available
-        - v1-dep-{{ .Branch }}-
-        # Default branch if not
-        - v1-dep-dev-
-        # Any branch if there are none on the default branch - this should be unnecessary if you have your default branch configured correctly
-        - v1-dep-
-    # This is based on your 1.0 configuration file or project settings
-    - run: sudo dpkg --add-architecture i386
-    - run: sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test; sudo apt-get -y -qq update
-    - run: sudo apt-get -y install gcc-powerpc-linux-gnu gcc-arm-linux-gnueabi libc6-dev-armel-cross gcc-aarch64-linux-gnu libc6-dev-arm64-cross
-    # Save dependency cache
-    - save_cache:
-        key: v1-dep-{{ .Branch }}-{{ epoch }}
-        paths:
-        # This is a broad list of cache paths to include many possible development environments
-        # You can probably delete some of these entries
-        - vendor/bundle
-        - ~/virtualenvs
-        - ~/.m2
-        - ~/.ivy2
-        - ~/.bundle
-        - ~/.go_workspace
-        - ~/.gradle
-        - ~/.cache/bower
-    # Test
-    #   This would typically be a build job when using workflows, possibly combined with build
-    # This is based on your 1.0 configuration file or project settings
-    - run: |
-        if [[ "$CIRCLE_NODE_INDEX" == "0" ]]                                    ; then cc -v; CFLAGS="-O0 -Werror" make all && make clean; fi &&
-        if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make gnu90build   && make clean; fi
-    - run: |
-        if [[ "$CIRCLE_NODE_INDEX" == "0" ]]                                    ; then make c99build     && make clean; fi &&
-        if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make gnu99build   && make clean; fi
-    - run: |
-        if [[ "$CIRCLE_NODE_INDEX" == "0" ]]                                    ; then make c11build     && make clean; fi &&
-        if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make ppc64build   && make clean; fi
-    - run: |
-        if [[ "$CIRCLE_NODE_INDEX" == "0" ]]                                    ; then make aarch64build && make clean; fi &&
-        if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make ppcbuild     && make clean; fi
-    - run: |
-        if [[ "$CIRCLE_NODE_INDEX" == "0" ]]                                    ; then make -j regressiontest && make clean; fi &&
-        if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make armbuild     && make clean; fi
-    - run: |
-        if [[ "$CIRCLE_NODE_INDEX" == "0" ]]                                    ; then make shortest     && make clean; fi &&
-        if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make -C tests test-legacy test-longmatch test-symbols && make clean; fi
-    - run: |
-        if [[ "$CIRCLE_NODE_INDEX" == "0" ]]                                    ; then make cxxtest      && make clean; fi &&
-        if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make -C lib libzstd-nomt && make clean; fi
-    # This is based on your 1.0 configuration file or project settings
-    - run: echo Circle CI tests finished
-    # Teardown
-    #   If you break your build into multiple jobs with workflows, you will probably want to do the parts of this that are relevant in each
-    # Save test results
-    - store_test_results:
-        path: /tmp/circleci-test-results
-    # Save artifacts
-    - store_artifacts:
-        path: /tmp/circleci-artifacts
-    - store_artifacts:
-        path: /tmp/circleci-test-results
+      - checkout
+      - run:
+          name: Install dependencies
+          command: |
+            apk add -q gzip coreutils
+      - run:
+          name: Publish
+          command: |
+            export VERSION=$(echo $CIRCLE_TAG | tail -c +2)
+            export ZSTD_VERSION=zstd-$VERSION
+            git archive $CIRCLE_TAG --prefix $ZSTD_VERSION/ --format tar \
+                        -o $ZSTD_VERSION.tar
+            gzip -9 $ZSTD_VERSION.tar
+            sha256sum $ZSTD_VERSION.tar.gz > $ZSTD_VERSION.tar.gz.sha256sum
+            mkdir -p $CIRCLE_ARTIFACTS
+            cp $ZSTD_VERSION.tar.gz{,.sha256sum} $CIRCLE_ARTIFACTS
+      - store_artifacts:
+          path: /tmp/circleci-artifacts
+
+workflows:
+  version: 2
+  commit:
+    jobs:
+      # Run the tests in parallel
+      - short-tests-0:
+          filters:
+            tags:
+              only: /.*/
+      - short-tests-1:
+          filters:
+            tags:
+              only: /.*/
+      # Only run on release tags.
+      - publish-github-release:
+          requires:
+            - short-tests-0
+            - short-tests-1
+          filters:
+            branches:
+              ignore: /.*/
+            tags:
+              only: /^v\d+\.\d+\.\d+$/
 
   # Longer tests
     #- make -C tests test-zstd-nolegacy && make clean
diff --git a/tests/fuzz/fuzz.py b/tests/fuzz/fuzz.py
index 84c28c63b..8ce293a3a 100755
--- a/tests/fuzz/fuzz.py
+++ b/tests/fuzz/fuzz.py
@@ -13,6 +13,7 @@ import argparse
 import contextlib
 import os
 import re
+import shlex
 import shutil
 import subprocess
 import sys
@@ -349,11 +350,11 @@ def build(args):
     targets = args.TARGET
     cc = args.cc
     cxx = args.cxx
-    cppflags = [args.cppflags]
-    cflags = [args.cflags]
-    ldflags = [args.ldflags]
-    cxxflags = [args.cxxflags]
-    mflags = [args.mflags] if args.mflags else []
+    cppflags = shlex.split(args.cppflags)
+    cflags = shlex.split(args.cflags)
+    ldflags = shlex.split(args.ldflags)
+    cxxflags = shlex.split(args.cxxflags)
+    mflags = shlex.split(args.mflags)
     # Flags to be added to both cflags and cxxflags
     common_flags = []
 

From f98c69d77cd9b1dde60df2de05613006842d4896 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Wed, 26 Sep 2018 14:24:28 -0700
Subject: [PATCH 301/372] fix : huge (>4GB) stream of blocks

experimental function ZSTD_compressBlock() is designed for very small data in mind,
for situation where saving the ~12 bytes of frame header can actually make a difference.

Some systems though may have to deal with small and large data entangled.
If it's larger than a block (> 128KB), compressBlock() cannot compress them in one round.

That's why it's possible to compress in multiple rounds.
This is a chain of compressed blocks.

Some users push this capability to the limit, encoding gigantic chain of blocks.
On crossing the 4GB limit, some internal overflow occurs.

This fix moves the overflow correction mechanism higher in the call chain,
so that it's applied also to gigantic chains of blocks.

Added a test case in fuzzer.c, which crashes before the fix, and pass now.
---
 lib/compress/zstd_compress.c | 27 ++++++++++++++-------------
 lib/compress/zstd_fast.c     |  1 +
 tests/fuzzer.c               | 14 ++++++++++++++
 3 files changed, 29 insertions(+), 13 deletions(-)

diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index 98e9917ab..714f2a7e4 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -2454,19 +2454,6 @@ static size_t ZSTD_compress_frameChunk (ZSTD_CCtx* cctx,
             return ERROR(dstSize_tooSmall);   /* not enough space to store compressed block */
         if (remaining < blockSize) blockSize = remaining;
 
-        if (ZSTD_window_needOverflowCorrection(ms->window, ip + blockSize)) {
-            U32 const cycleLog = ZSTD_cycleLog(cctx->appliedParams.cParams.chainLog, cctx->appliedParams.cParams.strategy);
-            U32 const correction = ZSTD_window_correctOverflow(&ms->window, cycleLog, maxDist, ip);
-            ZSTD_STATIC_ASSERT(ZSTD_CHAINLOG_MAX <= 30);
-            ZSTD_STATIC_ASSERT(ZSTD_WINDOWLOG_MAX_32 <= 30);
-            ZSTD_STATIC_ASSERT(ZSTD_WINDOWLOG_MAX <= 31);
-
-            ZSTD_reduceIndex(cctx, correction);
-            if (ms->nextToUpdate < correction) ms->nextToUpdate = 0;
-            else ms->nextToUpdate -= correction;
-            ms->loadedDictEnd = 0;
-            ms->dictMatchState = NULL;
-        }
         ZSTD_window_enforceMaxDist(&ms->window, ip + blockSize, maxDist, &ms->loadedDictEnd, &ms->dictMatchState);
         if (ms->nextToUpdate < ms->window.lowLimit) ms->nextToUpdate = ms->window.lowLimit;
 
@@ -2603,6 +2590,20 @@ static size_t ZSTD_compressContinue_internal (ZSTD_CCtx* cctx,
     if (cctx->appliedParams.ldmParams.enableLdm)
         ZSTD_window_update(&cctx->ldmState.window, src, srcSize);
 
+    if (ZSTD_window_needOverflowCorrection(ms->window, (const char*)src + srcSize)) {
+        U32 const cycleLog = ZSTD_cycleLog(cctx->appliedParams.cParams.chainLog, cctx->appliedParams.cParams.strategy);
+        U32 const correction = ZSTD_window_correctOverflow(&ms->window, cycleLog, 1 << cctx->appliedParams.cParams.windowLog, src);
+        ZSTD_STATIC_ASSERT(ZSTD_CHAINLOG_MAX <= 30);
+        ZSTD_STATIC_ASSERT(ZSTD_WINDOWLOG_MAX_32 <= 30);
+        ZSTD_STATIC_ASSERT(ZSTD_WINDOWLOG_MAX <= 31);
+
+        ZSTD_reduceIndex(cctx, correction);
+        if (ms->nextToUpdate < correction) ms->nextToUpdate = 0;
+        else ms->nextToUpdate -= correction;
+        ms->loadedDictEnd = 0;
+        ms->dictMatchState = NULL;
+    }
+
     DEBUGLOG(5, "ZSTD_compressContinue_internal (blockSize=%u)", (U32)cctx->blockSize);
     {   size_t const cSize = frame ?
                              ZSTD_compress_frameChunk (cctx, dst, dstCapacity, src, srcSize, lastFrameChunk) :
diff --git a/lib/compress/zstd_fast.c b/lib/compress/zstd_fast.c
index 7d7efb465..8d2c8452a 100644
--- a/lib/compress/zstd_fast.c
+++ b/lib/compress/zstd_fast.c
@@ -172,6 +172,7 @@ size_t ZSTD_compressBlock_fast_generic(
 
         if (ip <= ilimit) {
             /* Fill Table */
+            assert(base+current+2 > istart);  /* check base overflow */
             hashTable[ZSTD_hashPtr(base+current+2, hlog, mls)] = current+2;  /* here because current+2 could be > iend-8 */
             hashTable[ZSTD_hashPtr(ip-2, hlog, mls)] = (U32)(ip-2-base);
 
diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index 7191ffcbb..24e00e803 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -1291,6 +1291,20 @@ static int basicUnitTests(U32 seed, double compressibility)
           if (r != blockSize) goto _output_error; }
         DISPLAYLEVEL(3, "OK \n");
 
+        /* very long stream of block compression */
+        DISPLAYLEVEL(3, "test%3i : Huge block streaming compression test : ", testNb++);
+        CHECK( ZSTD_compressBegin(cctx, -99) );  /* we just want to quickly overflow internal U32 index */
+        CHECK( ZSTD_getBlockSize(cctx) >= blockSize);
+        {   U64 const toCompress = 5000000000ULL;   /* > 4 GB */
+            U64 compressed = 0;
+            while (compressed < toCompress) {
+                size_t const blockCSize = ZSTD_compressBlock(cctx, compressedBuffer, ZSTD_compressBound(blockSize), CNBuffer, blockSize);
+                if (ZSTD_isError(cSize)) goto _output_error;
+                compressed += blockCSize;
+            }
+        }
+        DISPLAYLEVEL(3, "OK \n");
+
         /* dictionary block compression */
         DISPLAYLEVEL(3, "test%3i : Dictionary Block compression test : ", testNb++);
         CHECK( ZSTD_compressBegin_usingDict(cctx, CNBuffer, dictSize, 5) );

From 0e2dbac18a2ba6ee073120f09b8b90f323bbfbc1 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Wed, 26 Sep 2018 15:35:38 -0700
Subject: [PATCH 302/372] changed overflow correction place

keep one in compress_frameChunk(),
so that it's tested at every loop
in case some user simply some large mulit-GB input in a single invocation.

Add one in ZSTD_compressBlock(),
since compressBlock() explicitly skips frameChunk().
---
 lib/compress/zstd_compress.c | 47 ++++++++++++++++++++++++------------
 1 file changed, 31 insertions(+), 16 deletions(-)

diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index 714f2a7e4..90c15291f 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -2333,6 +2333,7 @@ static size_t ZSTD_compressBlock_internal(ZSTD_CCtx* zc,
     ZSTD_matchState_t* const ms = &zc->blockState.matchState;
     DEBUGLOG(5, "ZSTD_compressBlock_internal (dstCapacity=%zu, dictLimit=%u, nextToUpdate=%u)",
                 dstCapacity, ms->window.dictLimit, ms->nextToUpdate);
+    assert(srcSize <= ZSTD_BLOCKSIZE_MAX);
 
     if (srcSize < MIN_CBLOCK_SIZE+ZSTD_blockHeaderSize+1) {
         ZSTD_ldm_skipSequences(&zc->externSeqStore, srcSize, zc->appliedParams.cParams.searchLength);
@@ -2454,7 +2455,19 @@ static size_t ZSTD_compress_frameChunk (ZSTD_CCtx* cctx,
             return ERROR(dstSize_tooSmall);   /* not enough space to store compressed block */
         if (remaining < blockSize) blockSize = remaining;
 
-        ZSTD_window_enforceMaxDist(&ms->window, ip + blockSize, maxDist, &ms->loadedDictEnd, &ms->dictMatchState);
+        if (ZSTD_window_needOverflowCorrection(ms->window, ip + blockSize)) {
+            U32 const cycleLog = ZSTD_cycleLog(cctx->appliedParams.cParams.chainLog, cctx->appliedParams.cParams.strategy);
+            U32 const correction = ZSTD_window_correctOverflow(&ms->window, cycleLog, maxDist, ip);
+            ZSTD_STATIC_ASSERT(ZSTD_CHAINLOG_MAX <= 30);
+            ZSTD_STATIC_ASSERT(ZSTD_WINDOWLOG_MAX_32 <= 30);
+            ZSTD_STATIC_ASSERT(ZSTD_WINDOWLOG_MAX <= 31);
+             ZSTD_reduceIndex(cctx, correction);
+            if (ms->nextToUpdate < correction) ms->nextToUpdate = 0;
+            else ms->nextToUpdate -= correction;
+            ms->loadedDictEnd = 0;
+            ms->dictMatchState = NULL;
+        }
+                ZSTD_window_enforceMaxDist(&ms->window, ip + blockSize, maxDist, &ms->loadedDictEnd, &ms->dictMatchState);
         if (ms->nextToUpdate < ms->window.lowLimit) ms->nextToUpdate = ms->window.lowLimit;
 
         {   size_t cSize = ZSTD_compressBlock_internal(cctx,
@@ -2566,7 +2579,7 @@ static size_t ZSTD_compressContinue_internal (ZSTD_CCtx* cctx,
                         const void* src, size_t srcSize,
                                U32 frame, U32 lastFrameChunk)
 {
-    ZSTD_matchState_t* ms = &cctx->blockState.matchState;
+    ZSTD_matchState_t* const ms = &cctx->blockState.matchState;
     size_t fhSize = 0;
 
     DEBUGLOG(5, "ZSTD_compressContinue_internal, stage: %u, srcSize: %u",
@@ -2590,20 +2603,6 @@ static size_t ZSTD_compressContinue_internal (ZSTD_CCtx* cctx,
     if (cctx->appliedParams.ldmParams.enableLdm)
         ZSTD_window_update(&cctx->ldmState.window, src, srcSize);
 
-    if (ZSTD_window_needOverflowCorrection(ms->window, (const char*)src + srcSize)) {
-        U32 const cycleLog = ZSTD_cycleLog(cctx->appliedParams.cParams.chainLog, cctx->appliedParams.cParams.strategy);
-        U32 const correction = ZSTD_window_correctOverflow(&ms->window, cycleLog, 1 << cctx->appliedParams.cParams.windowLog, src);
-        ZSTD_STATIC_ASSERT(ZSTD_CHAINLOG_MAX <= 30);
-        ZSTD_STATIC_ASSERT(ZSTD_WINDOWLOG_MAX_32 <= 30);
-        ZSTD_STATIC_ASSERT(ZSTD_WINDOWLOG_MAX <= 31);
-
-        ZSTD_reduceIndex(cctx, correction);
-        if (ms->nextToUpdate < correction) ms->nextToUpdate = 0;
-        else ms->nextToUpdate -= correction;
-        ms->loadedDictEnd = 0;
-        ms->dictMatchState = NULL;
-    }
-
     DEBUGLOG(5, "ZSTD_compressContinue_internal (blockSize=%u)", (U32)cctx->blockSize);
     {   size_t const cSize = frame ?
                              ZSTD_compress_frameChunk (cctx, dst, dstCapacity, src, srcSize, lastFrameChunk) :
@@ -2642,8 +2641,24 @@ size_t ZSTD_getBlockSize(const ZSTD_CCtx* cctx)
 
 size_t ZSTD_compressBlock(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize)
 {
+    ZSTD_matchState_t* const ms = &cctx->blockState.matchState;
     size_t const blockSizeMax = ZSTD_getBlockSize(cctx);
     if (srcSize > blockSizeMax) return ERROR(srcSize_wrong);
+
+    if (ZSTD_window_needOverflowCorrection(ms->window, (const char*)src + srcSize)) {
+        U32 const cycleLog = ZSTD_cycleLog(cctx->appliedParams.cParams.chainLog, cctx->appliedParams.cParams.strategy);
+        U32 const correction = ZSTD_window_correctOverflow(&ms->window, cycleLog, 1 << cctx->appliedParams.cParams.windowLog, src);
+        ZSTD_STATIC_ASSERT(ZSTD_CHAINLOG_MAX <= 30);
+        ZSTD_STATIC_ASSERT(ZSTD_WINDOWLOG_MAX_32 <= 30);
+        ZSTD_STATIC_ASSERT(ZSTD_WINDOWLOG_MAX <= 31);
+
+        ZSTD_reduceIndex(cctx, correction);
+        if (ms->nextToUpdate < correction) ms->nextToUpdate = 0;
+        else ms->nextToUpdate -= correction;
+        ms->loadedDictEnd = 0;
+        ms->dictMatchState = NULL;
+    }
+
     return ZSTD_compressContinue_internal(cctx, dst, dstCapacity, src, srcSize, 0 /* frame mode */, 0 /* last chunk */);
 }
 

From 404a7bfed0cad56ef9700fb746c15e85e5544467 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Wed, 26 Sep 2018 18:06:53 -0700
Subject: [PATCH 303/372] moved again overflow correction

cannot work from within ZSTD_compressBlock()
---
 lib/compress/zstd_compress.c | 38 +++++++++++++++++++-----------------
 1 file changed, 20 insertions(+), 18 deletions(-)

diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index 90c15291f..a969bd31c 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -2461,13 +2461,13 @@ static size_t ZSTD_compress_frameChunk (ZSTD_CCtx* cctx,
             ZSTD_STATIC_ASSERT(ZSTD_CHAINLOG_MAX <= 30);
             ZSTD_STATIC_ASSERT(ZSTD_WINDOWLOG_MAX_32 <= 30);
             ZSTD_STATIC_ASSERT(ZSTD_WINDOWLOG_MAX <= 31);
-             ZSTD_reduceIndex(cctx, correction);
+            ZSTD_reduceIndex(cctx, correction);
             if (ms->nextToUpdate < correction) ms->nextToUpdate = 0;
             else ms->nextToUpdate -= correction;
             ms->loadedDictEnd = 0;
             ms->dictMatchState = NULL;
         }
-                ZSTD_window_enforceMaxDist(&ms->window, ip + blockSize, maxDist, &ms->loadedDictEnd, &ms->dictMatchState);
+        ZSTD_window_enforceMaxDist(&ms->window, ip + blockSize, maxDist, &ms->loadedDictEnd, &ms->dictMatchState);
         if (ms->nextToUpdate < ms->window.lowLimit) ms->nextToUpdate = ms->window.lowLimit;
 
         {   size_t cSize = ZSTD_compressBlock_internal(cctx,
@@ -2600,8 +2600,25 @@ static size_t ZSTD_compressContinue_internal (ZSTD_CCtx* cctx,
     if (!ZSTD_window_update(&ms->window, src, srcSize)) {
         ms->nextToUpdate = ms->window.dictLimit;
     }
-    if (cctx->appliedParams.ldmParams.enableLdm)
+    if (cctx->appliedParams.ldmParams.enableLdm) {
         ZSTD_window_update(&cctx->ldmState.window, src, srcSize);
+    }
+
+    if (!frame) {
+        /* overflow check and correction for block mode */
+        if (ZSTD_window_needOverflowCorrection(ms->window, (const char*)src + srcSize)) {
+            U32 const cycleLog = ZSTD_cycleLog(cctx->appliedParams.cParams.chainLog, cctx->appliedParams.cParams.strategy);
+            U32 const correction = ZSTD_window_correctOverflow(&ms->window, cycleLog, 1 << cctx->appliedParams.cParams.windowLog, src);
+            ZSTD_STATIC_ASSERT(ZSTD_CHAINLOG_MAX <= 30);
+            ZSTD_STATIC_ASSERT(ZSTD_WINDOWLOG_MAX_32 <= 30);
+            ZSTD_STATIC_ASSERT(ZSTD_WINDOWLOG_MAX <= 31);
+            ZSTD_reduceIndex(cctx, correction);
+            if (ms->nextToUpdate < correction) ms->nextToUpdate = 0;
+            else ms->nextToUpdate -= correction;
+            ms->loadedDictEnd = 0;
+            ms->dictMatchState = NULL;
+        }
+    }
 
     DEBUGLOG(5, "ZSTD_compressContinue_internal (blockSize=%u)", (U32)cctx->blockSize);
     {   size_t const cSize = frame ?
@@ -2641,24 +2658,9 @@ size_t ZSTD_getBlockSize(const ZSTD_CCtx* cctx)
 
 size_t ZSTD_compressBlock(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize)
 {
-    ZSTD_matchState_t* const ms = &cctx->blockState.matchState;
     size_t const blockSizeMax = ZSTD_getBlockSize(cctx);
     if (srcSize > blockSizeMax) return ERROR(srcSize_wrong);
 
-    if (ZSTD_window_needOverflowCorrection(ms->window, (const char*)src + srcSize)) {
-        U32 const cycleLog = ZSTD_cycleLog(cctx->appliedParams.cParams.chainLog, cctx->appliedParams.cParams.strategy);
-        U32 const correction = ZSTD_window_correctOverflow(&ms->window, cycleLog, 1 << cctx->appliedParams.cParams.windowLog, src);
-        ZSTD_STATIC_ASSERT(ZSTD_CHAINLOG_MAX <= 30);
-        ZSTD_STATIC_ASSERT(ZSTD_WINDOWLOG_MAX_32 <= 30);
-        ZSTD_STATIC_ASSERT(ZSTD_WINDOWLOG_MAX <= 31);
-
-        ZSTD_reduceIndex(cctx, correction);
-        if (ms->nextToUpdate < correction) ms->nextToUpdate = 0;
-        else ms->nextToUpdate -= correction;
-        ms->loadedDictEnd = 0;
-        ms->dictMatchState = NULL;
-    }
-
     return ZSTD_compressContinue_internal(cctx, dst, dstCapacity, src, srcSize, 0 /* frame mode */, 0 /* last chunk */);
 }
 

From ca0cfa3dbd5616a4497ea4aabb7fc45f490527e4 Mon Sep 17 00:00:00 2001
From: Nick Terrell 
Date: Thu, 27 Sep 2018 12:48:29 -0700
Subject: [PATCH 304/372] [zstreamtest] Reduce memory of newapi tests

We could allocate up to 2^28 bytes of memory when using 2 threads with
window log = 24. Now, we limit it to 2^26 bytes of memory when not running
big tests.

I chose max window log = 22 since that is the maximum source size when
big tests are disabled. Hopefully this will be enough to reduce or
eliminate the test failures.
---
 tests/zstreamtest.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/zstreamtest.c b/tests/zstreamtest.c
index 96136a625..3959f5d7d 100644
--- a/tests/zstreamtest.c
+++ b/tests/zstreamtest.c
@@ -1808,7 +1808,7 @@ static int fuzzerTests_newAPI(U32 seed, U32 nbTests, unsigned startTest,
             }
             {   U64 const pledgedSrcSize = (FUZ_rand(&lseed) & 3) ? ZSTD_CONTENTSIZE_UNKNOWN : maxTestSize;
                 ZSTD_compressionParameters cParams = ZSTD_getCParams(cLevel, pledgedSrcSize, dictSize);
-                static const U32 windowLogMax = 24;
+                const U32 windowLogMax = bigTests ? 24 : 22;
                 if (dictSize)
                     DISPLAYLEVEL(5, "t%u: with dictionary of size : %zu \n", testNb, dictSize);
 

From 7ee910e86b5a4c6ee7479959df547b775ae169a1 Mon Sep 17 00:00:00 2001
From: Nick Terrell 
Date: Thu, 27 Sep 2018 13:55:24 -0700
Subject: [PATCH 305/372] More aggressive limitations

---
 tests/zstreamtest.c | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/tests/zstreamtest.c b/tests/zstreamtest.c
index 3959f5d7d..b56009323 100644
--- a/tests/zstreamtest.c
+++ b/tests/zstreamtest.c
@@ -1808,7 +1808,8 @@ static int fuzzerTests_newAPI(U32 seed, U32 nbTests, unsigned startTest,
             }
             {   U64 const pledgedSrcSize = (FUZ_rand(&lseed) & 3) ? ZSTD_CONTENTSIZE_UNKNOWN : maxTestSize;
                 ZSTD_compressionParameters cParams = ZSTD_getCParams(cLevel, pledgedSrcSize, dictSize);
-                const U32 windowLogMax = bigTests ? 24 : 22;
+                const U32 windowLogMax = bigTests ? 24 : 20;
+                const U32 searchLogMax = bigTests ? 15 : 13;
                 if (dictSize)
                     DISPLAYLEVEL(5, "t%u: with dictionary of size : %zu \n", testNb, dictSize);
 
@@ -1818,6 +1819,7 @@ static int fuzzerTests_newAPI(U32 seed, U32 nbTests, unsigned startTest,
                 cParams.hashLog += (FUZ_rand(&lseed) & 3) - 1;
                 cParams.chainLog += (FUZ_rand(&lseed) & 3) - 1;
                 cParams.searchLog += (FUZ_rand(&lseed) & 3) - 1;
+                cParams.searchLog = MIN(searchLogMax, cParams.searchLog);
                 cParams.searchLength += (FUZ_rand(&lseed) & 3) - 1;
                 cParams.targetLength = (U32)((cParams.targetLength + 1 ) * (0.5 + ((double)(FUZ_rand(&lseed) & 127) / 128)));
                 cParams = ZSTD_adjustCParams(cParams, pledgedSrcSize, dictSize);

From f2d6db45cd28457fa08467416e8535985f062859 Mon Sep 17 00:00:00 2001
From: Nick Terrell 
Date: Thu, 27 Sep 2018 15:13:43 -0700
Subject: [PATCH 306/372] [zstd] Add -Wmissing-prototypes

---
 lib/Makefile                     |  2 +-
 lib/compress/fse_compress.c      | 13 -------------
 lib/compress/huf_compress.c      |  2 +-
 lib/compress/zstd_compress.c     | 25 ++++++++-----------------
 lib/compress/zstd_lazy.c         |  2 +-
 lib/compress/zstd_opt.c          |  2 +-
 lib/compress/zstdmt_compress.c   |  2 +-
 lib/decompress/zstd_decompress.c |  6 ++++++
 lib/dictBuilder/zdict.c          | 13 +++++++++----
 lib/legacy/zstd_v01.c            |  6 +++---
 lib/legacy/zstd_v04.c            |  5 -----
 lib/legacy/zstd_v05.c            |  9 +++++----
 lib/legacy/zstd_v06.c            | 17 ++++++++---------
 lib/legacy/zstd_v07.c            | 18 +++++++++---------
 programs/Makefile                |  2 +-
 programs/datagen.c               |  3 ++-
 programs/dibio.c                 |  4 ----
 programs/fileio.c                |  2 +-
 tests/Makefile                   |  2 +-
 tests/fullbench.c                | 22 +++++++++++-----------
 tests/fuzzer.c                   |  2 ++
 tests/paramgrill.c               |  6 +++---
 tests/zstreamtest.c              |  4 ++--
 23 files changed, 76 insertions(+), 93 deletions(-)

diff --git a/lib/Makefile b/lib/Makefile
index a161809cb..3502464b8 100644
--- a/lib/Makefile
+++ b/lib/Makefile
@@ -27,7 +27,7 @@ DEBUGFLAGS= -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \
             -Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \
             -Wstrict-prototypes -Wundef -Wpointer-arith -Wformat-security \
             -Wvla -Wformat=2 -Winit-self -Wfloat-equal -Wwrite-strings \
-            -Wredundant-decls
+            -Wredundant-decls -Wmissing-prototypes
 CFLAGS  += $(DEBUGFLAGS) $(MOREFLAGS)
 FLAGS    = $(CPPFLAGS) $(CFLAGS)
 
diff --git a/lib/compress/fse_compress.c b/lib/compress/fse_compress.c
index 4075db217..4408f0ed5 100644
--- a/lib/compress/fse_compress.c
+++ b/lib/compress/fse_compress.c
@@ -321,19 +321,6 @@ size_t FSE_writeNCount (void* buffer, size_t bufferSize,
 /*-**************************************************************
 *  FSE Compression Code
 ****************************************************************/
-/*! FSE_sizeof_CTable() :
-    FSE_CTable is a variable size structure which contains :
-    `U16 tableLog;`
-    `U16 maxSymbolValue;`
-    `U16 nextStateNumber[1 << tableLog];`                         // This size is variable
-    `FSE_symbolCompressionTransform symbolTT[maxSymbolValue+1];`  // This size is variable
-Allocation is manual (C standard does not support variable-size structures).
-*/
-size_t FSE_sizeof_CTable (unsigned maxSymbolValue, unsigned tableLog)
-{
-    if (tableLog > FSE_MAX_TABLELOG) return ERROR(tableLog_tooLarge);
-    return FSE_CTABLE_SIZE_U32 (tableLog, maxSymbolValue) * sizeof(U32);
-}
 
 FSE_CTable* FSE_createCTable (unsigned maxSymbolValue, unsigned tableLog)
 {
diff --git a/lib/compress/huf_compress.c b/lib/compress/huf_compress.c
index 9cdaa5d79..4c40572f2 100644
--- a/lib/compress/huf_compress.c
+++ b/lib/compress/huf_compress.c
@@ -82,7 +82,7 @@ unsigned HUF_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxS
  * Note : all elements within weightTable are supposed to be <= HUF_TABLELOG_MAX.
  */
 #define MAX_FSE_TABLELOG_FOR_HUFF_HEADER 6
-size_t HUF_compressWeights (void* dst, size_t dstSize, const void* weightTable, size_t wtSize)
+static size_t HUF_compressWeights (void* dst, size_t dstSize, const void* weightTable, size_t wtSize)
 {
     BYTE* const ostart = (BYTE*) dst;
     BYTE* op = ostart;
diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index 18c7cc14b..fcdf93f57 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -1531,15 +1531,6 @@ static void ZSTD_reduceIndex (ZSTD_CCtx* zc, const U32 reducerValue)
 
 /* See doc/zstd_compression_format.md for detailed format description */
 
-size_t ZSTD_noCompressBlock (void* dst, size_t dstCapacity, const void* src, size_t srcSize)
-{
-    if (srcSize + ZSTD_blockHeaderSize > dstCapacity) return ERROR(dstSize_tooSmall);
-    memcpy((BYTE*)dst + ZSTD_blockHeaderSize, src, srcSize);
-    MEM_writeLE24(dst, (U32)(srcSize << 2) + (U32)bt_raw);
-    return ZSTD_blockHeaderSize+srcSize;
-}
-
-
 static size_t ZSTD_noCompressLiterals (void* dst, size_t dstCapacity, const void* src, size_t srcSize)
 {
     BYTE* const ostart = (BYTE* const)dst;
@@ -2091,7 +2082,7 @@ ZSTD_encodeSequences_bmi2(
 
 #endif
 
-size_t ZSTD_encodeSequences(
+static size_t ZSTD_encodeSequences(
             void* dst, size_t dstCapacity,
             FSE_CTable const* CTable_MatchLength, BYTE const* mlCodeTable,
             FSE_CTable const* CTable_OffsetBits, BYTE const* ofCodeTable,
@@ -2883,13 +2874,13 @@ ZSTD_compress_insertDictionary(ZSTD_compressedBlockState_t* bs,
 
 /*! ZSTD_compressBegin_internal() :
  * @return : 0, or an error code */
-size_t ZSTD_compressBegin_internal(ZSTD_CCtx* cctx,
-                             const void* dict, size_t dictSize,
-                             ZSTD_dictContentType_e dictContentType,
-                             ZSTD_dictTableLoadMethod_e dtlm,
-                             const ZSTD_CDict* cdict,
-                             ZSTD_CCtx_params params, U64 pledgedSrcSize,
-                             ZSTD_buffered_policy_e zbuff)
+static size_t ZSTD_compressBegin_internal(ZSTD_CCtx* cctx,
+                                    const void* dict, size_t dictSize,
+                                    ZSTD_dictContentType_e dictContentType,
+                                    ZSTD_dictTableLoadMethod_e dtlm,
+                                    const ZSTD_CDict* cdict,
+                                    ZSTD_CCtx_params params, U64 pledgedSrcSize,
+                                    ZSTD_buffered_policy_e zbuff)
 {
     DEBUGLOG(4, "ZSTD_compressBegin_internal: wlog=%u", params.cParams.windowLog);
     /* params are supposed to be fully validated at this point */
diff --git a/lib/compress/zstd_lazy.c b/lib/compress/zstd_lazy.c
index bfe944928..f17fced15 100644
--- a/lib/compress/zstd_lazy.c
+++ b/lib/compress/zstd_lazy.c
@@ -16,7 +16,7 @@
 *  Binary Tree search
 ***************************************/
 
-void ZSTD_updateDUBT(
+static void ZSTD_updateDUBT(
                 ZSTD_matchState_t* ms, ZSTD_compressionParameters const* cParams,
                 const BYTE* ip, const BYTE* iend,
                 U32 mls)
diff --git a/lib/compress/zstd_opt.c b/lib/compress/zstd_opt.c
index c4b9bb13b..694deec78 100644
--- a/lib/compress/zstd_opt.c
+++ b/lib/compress/zstd_opt.c
@@ -741,7 +741,7 @@ typedef struct repcodes_s {
     U32 rep[3];
 } repcodes_t;
 
-repcodes_t ZSTD_updateRep(U32 const rep[3], U32 const offset, U32 const ll0)
+static repcodes_t ZSTD_updateRep(U32 const rep[3], U32 const offset, U32 const ll0)
 {
     repcodes_t newReps;
     if (offset >= ZSTD_REP_NUM) {  /* full offset */
diff --git a/lib/compress/zstdmt_compress.c b/lib/compress/zstdmt_compress.c
index 6b9c24b56..f4aba1d2c 100644
--- a/lib/compress/zstdmt_compress.c
+++ b/lib/compress/zstdmt_compress.c
@@ -640,7 +640,7 @@ typedef struct {
 }
 
 /* ZSTDMT_compressionJob() is a POOL_function type */
-void ZSTDMT_compressionJob(void* jobDescription)
+static void ZSTDMT_compressionJob(void* jobDescription)
 {
     ZSTDMT_jobDescription* const job = (ZSTDMT_jobDescription*)jobDescription;
     ZSTD_CCtx_params jobParams = job->params;   /* do not modify job->params ! copy it, modify the copy */
diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c
index 1382c9c7f..e7e91b180 100644
--- a/lib/decompress/zstd_decompress.c
+++ b/lib/decompress/zstd_decompress.c
@@ -568,6 +568,9 @@ static size_t ZSTD_setRleBlock(void* dst, size_t dstCapacity,
     return regenSize;
 }
 
+/* Hidden declaration for fullbench */
+size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx,
+                          const void* src, size_t srcSize);
 /*! ZSTD_decodeLiteralsBlock() :
  * @return : nb of bytes read from src (< srcSize )
  *  note : symbol not declared but exposed for fullbench */
@@ -972,6 +975,9 @@ static const U32 ML_base[MaxML+1] = {
                     67, 83, 99, 0x83, 0x103, 0x203, 0x403, 0x803,
                     0x1003, 0x2003, 0x4003, 0x8003, 0x10003 };
 
+/* Hidden delcaration for fullbench */
+size_t ZSTD_decodeSeqHeaders(ZSTD_DCtx* dctx, int* nbSeqPtr,
+                             const void* src, size_t srcSize);
 
 size_t ZSTD_decodeSeqHeaders(ZSTD_DCtx* dctx, int* nbSeqPtr,
                              const void* src, size_t srcSize)
diff --git a/lib/dictBuilder/zdict.c b/lib/dictBuilder/zdict.c
index eb1a5b669..2964b69ff 100644
--- a/lib/dictBuilder/zdict.c
+++ b/lib/dictBuilder/zdict.c
@@ -910,9 +910,10 @@ size_t ZDICT_finalizeDictionary(void* dictBuffer, size_t dictBufferCapacity,
 }
 
 
-size_t ZDICT_addEntropyTablesFromBuffer_advanced(void* dictBuffer, size_t dictContentSize, size_t dictBufferCapacity,
-                                                 const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples,
-                                                 ZDICT_params_t params)
+static size_t ZDICT_addEntropyTablesFromBuffer_advanced(
+        void* dictBuffer, size_t dictContentSize, size_t dictBufferCapacity,
+        const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples,
+        ZDICT_params_t params)
 {
     int const compressionLevel = (params.compressionLevel == 0) ? g_compressionLevel_default : params.compressionLevel;
     U32 const notificationLevel = params.notificationLevel;
@@ -943,7 +944,11 @@ size_t ZDICT_addEntropyTablesFromBuffer_advanced(void* dictBuffer, size_t dictCo
     return MIN(dictBufferCapacity, hSize+dictContentSize);
 }
 
-
+/* Hidden declaration for dbio.c */
+size_t ZDICT_trainFromBuffer_unsafe_legacy(
+                            void* dictBuffer, size_t maxDictSize,
+                            const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples,
+                            ZDICT_legacy_params_t params);
 /*! ZDICT_trainFromBuffer_unsafe_legacy() :
 *   Warning : `samplesBuffer` must be followed by noisy guard band.
 *   @return : size of dictionary, or an error code which can be tested with ZDICT_isError()
diff --git a/lib/legacy/zstd_v01.c b/lib/legacy/zstd_v01.c
index ae1cb2ce5..6303e1cfc 100644
--- a/lib/legacy/zstd_v01.c
+++ b/lib/legacy/zstd_v01.c
@@ -1458,7 +1458,7 @@ unsigned ZSTDv01_isError(size_t code) { return ERR_isError(code); }
 *   Decompression code
 **************************************************************/
 
-size_t ZSTDv01_getcBlockSize(const void* src, size_t srcSize, blockProperties_t* bpPtr)
+static size_t ZSTDv01_getcBlockSize(const void* src, size_t srcSize, blockProperties_t* bpPtr)
 {
     const BYTE* const in = (const BYTE* const)src;
     BYTE headerFlags;
@@ -1511,7 +1511,7 @@ static size_t ZSTD_decompressLiterals(void* ctx,
 }
 
 
-size_t ZSTDv01_decodeLiteralsBlock(void* ctx,
+static size_t ZSTDv01_decodeLiteralsBlock(void* ctx,
                                 void* dst, size_t maxDstSize,
                           const BYTE** litStart, size_t* litSize,
                           const void* src, size_t srcSize)
@@ -1563,7 +1563,7 @@ size_t ZSTDv01_decodeLiteralsBlock(void* ctx,
 }
 
 
-size_t ZSTDv01_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* dumpsLengthPtr,
+static size_t ZSTDv01_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* dumpsLengthPtr,
                          FSE_DTable* DTableLL, FSE_DTable* DTableML, FSE_DTable* DTableOffb,
                          const void* src, size_t srcSize)
 {
diff --git a/lib/legacy/zstd_v04.c b/lib/legacy/zstd_v04.c
index 15000db6d..e852bb911 100644
--- a/lib/legacy/zstd_v04.c
+++ b/lib/legacy/zstd_v04.c
@@ -3622,8 +3622,3 @@ size_t ZBUFFv04_decompressContinue(ZBUFFv04_DCtx* dctx, void* dst, size_t* maxDs
 
 ZSTD_DCtx* ZSTDv04_createDCtx(void) { return ZSTD_createDCtx(); }
 size_t ZSTDv04_freeDCtx(ZSTD_DCtx* dctx) { return ZSTD_freeDCtx(dctx); }
-
-size_t ZSTDv04_getFrameParams(ZSTD_parameters* params, const void* src, size_t srcSize)
-{
-    return ZSTD_getFrameParams(params, src, srcSize);
-}
diff --git a/lib/legacy/zstd_v05.c b/lib/legacy/zstd_v05.c
index 65f07a3b5..6f4dc72b7 100644
--- a/lib/legacy/zstd_v05.c
+++ b/lib/legacy/zstd_v05.c
@@ -2659,6 +2659,7 @@ struct ZSTDv05_DCtx_s
     BYTE headerBuffer[ZSTDv05_frameHeaderSize_max];
 };  /* typedef'd to ZSTDv05_DCtx within "zstd_static.h" */
 
+size_t ZSTDv05_sizeofDCtx (void); /* Hidden declaration */
 size_t ZSTDv05_sizeofDCtx (void) { return sizeof(ZSTDv05_DCtx); }
 
 size_t ZSTDv05_decompressBegin(ZSTDv05_DCtx* dctx)
@@ -2823,7 +2824,7 @@ static size_t ZSTDv05_decodeFrameHeader_Part2(ZSTDv05_DCtx* zc, const void* src,
 }
 
 
-size_t ZSTDv05_getcBlockSize(const void* src, size_t srcSize, blockProperties_t* bpPtr)
+static size_t ZSTDv05_getcBlockSize(const void* src, size_t srcSize, blockProperties_t* bpPtr)
 {
     const BYTE* const in = (const BYTE* const)src;
     BYTE headerFlags;
@@ -2855,8 +2856,8 @@ static size_t ZSTDv05_copyRawBlock(void* dst, size_t maxDstSize, const void* src
 
 /*! ZSTDv05_decodeLiteralsBlock() :
     @return : nb of bytes read from src (< srcSize ) */
-size_t ZSTDv05_decodeLiteralsBlock(ZSTDv05_DCtx* dctx,
-                          const void* src, size_t srcSize)   /* note : srcSize < BLOCKSIZE */
+static size_t ZSTDv05_decodeLiteralsBlock(ZSTDv05_DCtx* dctx,
+                                    const void* src, size_t srcSize)   /* note : srcSize < BLOCKSIZE */
 {
     const BYTE* const istart = (const BYTE*) src;
 
@@ -2990,7 +2991,7 @@ size_t ZSTDv05_decodeLiteralsBlock(ZSTDv05_DCtx* dctx,
 }
 
 
-size_t ZSTDv05_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* dumpsLengthPtr,
+static size_t ZSTDv05_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* dumpsLengthPtr,
                          FSEv05_DTable* DTableLL, FSEv05_DTable* DTableML, FSEv05_DTable* DTableOffb,
                          const void* src, size_t srcSize, U32 flagStaticTable)
 {
diff --git a/lib/legacy/zstd_v06.c b/lib/legacy/zstd_v06.c
index ca982c35e..60d8d6fd9 100644
--- a/lib/legacy/zstd_v06.c
+++ b/lib/legacy/zstd_v06.c
@@ -1250,9 +1250,7 @@ const char* FSEv06_getErrorName(size_t code) { return ERR_getErrorName(code); }
 /* **************************************************************
 *  HUF Error Management
 ****************************************************************/
-unsigned HUFv06_isError(size_t code) { return ERR_isError(code); }
-
-const char* HUFv06_getErrorName(size_t code) { return ERR_getErrorName(code); }
+static unsigned HUFv06_isError(size_t code) { return ERR_isError(code); }
 
 
 /*-**************************************************************
@@ -2823,7 +2821,8 @@ struct ZSTDv06_DCtx_s
     BYTE headerBuffer[ZSTDv06_FRAMEHEADERSIZE_MAX];
 };  /* typedef'd to ZSTDv06_DCtx within "zstd_static.h" */
 
-size_t ZSTDv06_sizeofDCtx (void) { return sizeof(ZSTDv06_DCtx); }   /* non published interface */
+size_t ZSTDv06_sizeofDCtx (void); /* Hidden declaration */
+size_t ZSTDv06_sizeofDCtx (void) { return sizeof(ZSTDv06_DCtx); }
 
 size_t ZSTDv06_decompressBegin(ZSTDv06_DCtx* dctx)
 {
@@ -3022,7 +3021,7 @@ typedef struct
 
 /*! ZSTDv06_getcBlockSize() :
 *   Provides the size of compressed block from block header `src` */
-size_t ZSTDv06_getcBlockSize(const void* src, size_t srcSize, blockProperties_t* bpPtr)
+static size_t ZSTDv06_getcBlockSize(const void* src, size_t srcSize, blockProperties_t* bpPtr)
 {
     const BYTE* const in = (const BYTE* const)src;
     U32 cSize;
@@ -3050,7 +3049,7 @@ static size_t ZSTDv06_copyRawBlock(void* dst, size_t dstCapacity, const void* sr
 
 /*! ZSTDv06_decodeLiteralsBlock() :
     @return : nb of bytes read from src (< srcSize ) */
-size_t ZSTDv06_decodeLiteralsBlock(ZSTDv06_DCtx* dctx,
+static size_t ZSTDv06_decodeLiteralsBlock(ZSTDv06_DCtx* dctx,
                           const void* src, size_t srcSize)   /* note : srcSize < BLOCKSIZE */
 {
     const BYTE* const istart = (const BYTE*) src;
@@ -3184,7 +3183,7 @@ size_t ZSTDv06_decodeLiteralsBlock(ZSTDv06_DCtx* dctx,
     @return : nb bytes read from src,
               or an error code if it fails, testable with ZSTDv06_isError()
 */
-size_t ZSTDv06_buildSeqTable(FSEv06_DTable* DTable, U32 type, U32 max, U32 maxLog,
+static size_t ZSTDv06_buildSeqTable(FSEv06_DTable* DTable, U32 type, U32 max, U32 maxLog,
                                  const void* src, size_t srcSize,
                                  const S16* defaultNorm, U32 defaultLog, U32 flagRepeatTable)
 {
@@ -3214,7 +3213,7 @@ size_t ZSTDv06_buildSeqTable(FSEv06_DTable* DTable, U32 type, U32 max, U32 maxLo
 }
 
 
-size_t ZSTDv06_decodeSeqHeaders(int* nbSeqPtr,
+static size_t ZSTDv06_decodeSeqHeaders(int* nbSeqPtr,
                              FSEv06_DTable* DTableLL, FSEv06_DTable* DTableML, FSEv06_DTable* DTableOffb, U32 flagRepeatTable,
                              const void* src, size_t srcSize)
 {
@@ -3359,7 +3358,7 @@ static void ZSTDv06_decodeSequence(seq_t* seq, seqState_t* seqState)
 }
 
 
-size_t ZSTDv06_execSequence(BYTE* op,
+static size_t ZSTDv06_execSequence(BYTE* op,
                                 BYTE* const oend, seq_t sequence,
                                 const BYTE** litPtr, const BYTE* const litLimit,
                                 const BYTE* const base, const BYTE* const vBase, const BYTE* const dictEnd)
diff --git a/lib/legacy/zstd_v07.c b/lib/legacy/zstd_v07.c
index 9f95f62ad..c7bb7a529 100644
--- a/lib/legacy/zstd_v07.c
+++ b/lib/legacy/zstd_v07.c
@@ -2628,7 +2628,7 @@ const char* ZBUFFv07_getErrorName(size_t errorCode) { return ERR_getErrorName(er
 
 
 
-void* ZSTDv07_defaultAllocFunction(void* opaque, size_t size)
+static void* ZSTDv07_defaultAllocFunction(void* opaque, size_t size)
 {
     void* address = malloc(size);
     (void)opaque;
@@ -2636,7 +2636,7 @@ void* ZSTDv07_defaultAllocFunction(void* opaque, size_t size)
     return address;
 }
 
-void ZSTDv07_defaultFreeFunction(void* opaque, void* address)
+static void ZSTDv07_defaultFreeFunction(void* opaque, void* address)
 {
     (void)opaque;
     /* if (address) printf("free %p opaque=%p \n", address, opaque); */
@@ -3250,7 +3250,7 @@ typedef struct
 
 /*! ZSTDv07_getcBlockSize() :
 *   Provides the size of compressed block from block header `src` */
-size_t ZSTDv07_getcBlockSize(const void* src, size_t srcSize, blockProperties_t* bpPtr)
+static size_t ZSTDv07_getcBlockSize(const void* src, size_t srcSize, blockProperties_t* bpPtr)
 {
     const BYTE* const in = (const BYTE* const)src;
     U32 cSize;
@@ -3277,7 +3277,7 @@ static size_t ZSTDv07_copyRawBlock(void* dst, size_t dstCapacity, const void* sr
 
 /*! ZSTDv07_decodeLiteralsBlock() :
     @return : nb of bytes read from src (< srcSize ) */
-size_t ZSTDv07_decodeLiteralsBlock(ZSTDv07_DCtx* dctx,
+static size_t ZSTDv07_decodeLiteralsBlock(ZSTDv07_DCtx* dctx,
                           const void* src, size_t srcSize)   /* note : srcSize < BLOCKSIZE */
 {
     const BYTE* const istart = (const BYTE*) src;
@@ -3411,7 +3411,7 @@ size_t ZSTDv07_decodeLiteralsBlock(ZSTDv07_DCtx* dctx,
     @return : nb bytes read from src,
               or an error code if it fails, testable with ZSTDv07_isError()
 */
-size_t ZSTDv07_buildSeqTable(FSEv07_DTable* DTable, U32 type, U32 max, U32 maxLog,
+static size_t ZSTDv07_buildSeqTable(FSEv07_DTable* DTable, U32 type, U32 max, U32 maxLog,
                                  const void* src, size_t srcSize,
                                  const S16* defaultNorm, U32 defaultLog, U32 flagRepeatTable)
 {
@@ -3441,7 +3441,7 @@ size_t ZSTDv07_buildSeqTable(FSEv07_DTable* DTable, U32 type, U32 max, U32 maxLo
 }
 
 
-size_t ZSTDv07_decodeSeqHeaders(int* nbSeqPtr,
+static size_t ZSTDv07_decodeSeqHeaders(int* nbSeqPtr,
                              FSEv07_DTable* DTableLL, FSEv07_DTable* DTableML, FSEv07_DTable* DTableOffb, U32 flagRepeatTable,
                              const void* src, size_t srcSize)
 {
@@ -3773,7 +3773,7 @@ ZSTDLIBv07_API size_t ZSTDv07_insertBlock(ZSTDv07_DCtx* dctx, const void* blockS
 }
 
 
-size_t ZSTDv07_generateNxBytes(void* dst, size_t dstCapacity, BYTE byte, size_t length)
+static size_t ZSTDv07_generateNxBytes(void* dst, size_t dstCapacity, BYTE byte, size_t length)
 {
     if (length > dstCapacity) return ERROR(dstSize_tooSmall);
     memset(dst, byte, length);
@@ -3853,7 +3853,7 @@ static size_t ZSTDv07_decompressFrame(ZSTDv07_DCtx* dctx,
 *   It avoids reloading the dictionary each time.
 *   `preparedDCtx` must have been properly initialized using ZSTDv07_decompressBegin_usingDict().
 *   Requires 2 contexts : 1 for reference (preparedDCtx), which will not be modified, and 1 to run the decompression operation (dctx) */
-size_t ZSTDv07_decompress_usingPreparedDCtx(ZSTDv07_DCtx* dctx, const ZSTDv07_DCtx* refDCtx,
+static size_t ZSTDv07_decompress_usingPreparedDCtx(ZSTDv07_DCtx* dctx, const ZSTDv07_DCtx* refDCtx,
                                          void* dst, size_t dstCapacity,
                                    const void* src, size_t srcSize)
 {
@@ -4148,7 +4148,7 @@ struct ZSTDv07_DDict_s {
     ZSTDv07_DCtx* refContext;
 };  /* typedef'd tp ZSTDv07_CDict within zstd.h */
 
-ZSTDv07_DDict* ZSTDv07_createDDict_advanced(const void* dict, size_t dictSize, ZSTDv07_customMem customMem)
+static ZSTDv07_DDict* ZSTDv07_createDDict_advanced(const void* dict, size_t dictSize, ZSTDv07_customMem customMem)
 {
     if (!customMem.customAlloc && !customMem.customFree)
         customMem = defaultCustomMem;
diff --git a/programs/Makefile b/programs/Makefile
index 7dc65118b..5948cf158 100644
--- a/programs/Makefile
+++ b/programs/Makefile
@@ -48,7 +48,7 @@ DEBUGFLAGS+=-Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \
             -Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \
             -Wstrict-prototypes -Wundef -Wpointer-arith -Wformat-security \
             -Wvla -Wformat=2 -Winit-self -Wfloat-equal -Wwrite-strings \
-            -Wredundant-decls
+            -Wredundant-decls -Wmissing-prototypes
 CFLAGS  += $(DEBUGFLAGS) $(MOREFLAGS)
 FLAGS    = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS)
 
diff --git a/programs/datagen.c b/programs/datagen.c
index a489d6af0..c83836584 100644
--- a/programs/datagen.c
+++ b/programs/datagen.c
@@ -13,6 +13,7 @@
 /*-************************************
 *  Dependencies
 **************************************/
+#include "datagen.h"
 #include "platform.h"  /* SET_BINARY_MODE */
 #include     /* malloc, free */
 #include      /* FILE, fwrite, fprintf */
@@ -91,7 +92,7 @@ static U32 RDG_randLength(unsigned* seedPtr)
     return (RDG_rand(seedPtr) & 0x1FF) + 0xF;
 }
 
-void RDG_genBlock(void* buffer, size_t buffSize, size_t prefixSize, double matchProba, const BYTE* ldt, unsigned* seedPtr)
+static void RDG_genBlock(void* buffer, size_t buffSize, size_t prefixSize, double matchProba, const BYTE* ldt, unsigned* seedPtr)
 {
     BYTE* const buffPtr = (BYTE*)buffer;
     U32 const matchProba32 = (U32)(32768 * matchProba);
diff --git a/programs/dibio.c b/programs/dibio.c
index 4b68be6c9..d3fd8cc05 100644
--- a/programs/dibio.c
+++ b/programs/dibio.c
@@ -84,10 +84,6 @@ static UTIL_time_t g_displayClock = UTIL_TIME_INITIALIZER;
 /* ********************************************************
 *  Helper functions
 **********************************************************/
-unsigned DiB_isError(size_t errorCode) { return ERR_isError(errorCode); }
-
-const char* DiB_getErrorName(size_t errorCode) { return ERR_getErrorName(errorCode); }
-
 #undef MIN
 #define MIN(a,b)    ((a) < (b) ? (a) : (b))
 
diff --git a/programs/fileio.c b/programs/fileio.c
index 53f72aa72..1b04f2ea0 100644
--- a/programs/fileio.c
+++ b/programs/fileio.c
@@ -1422,7 +1422,7 @@ static void FIO_zstdErrorHelp(dRess_t* ress, size_t err, char const* srcFileName
  *  @return : size of decoded zstd frame, or an error code
 */
 #define FIO_ERROR_FRAME_DECODING   ((unsigned long long)(-2))
-unsigned long long FIO_decompressZstdFrame(dRess_t* ress,
+static unsigned long long FIO_decompressZstdFrame(dRess_t* ress,
                                        FILE* finput,
                                        const char* srcFileName,
                                        U64 alreadyDecoded)
diff --git a/tests/Makefile b/tests/Makefile
index d0d798fc3..50c1a6156 100644
--- a/tests/Makefile
+++ b/tests/Makefile
@@ -35,7 +35,7 @@ CFLAGS     += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow                 \
               -Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \
               -Wstrict-prototypes -Wundef -Wformat-security                   \
               -Wvla -Wformat=2 -Winit-self -Wfloat-equal -Wwrite-strings      \
-              -Wredundant-decls
+              -Wredundant-decls -Wmissing-prototypes
 CFLAGS     += $(DEBUGFLAGS) $(MOREFLAGS)
 FLAGS       = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS)
 
diff --git a/tests/fullbench.c b/tests/fullbench.c
index fd4815c91..b05f1537c 100644
--- a/tests/fullbench.c
+++ b/tests/fullbench.c
@@ -127,14 +127,14 @@ static ZSTD_DCtx* g_zdc = NULL;
 
 #ifndef ZSTD_DLL_IMPORT
 extern size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* ctx, const void* src, size_t srcSize);
-size_t local_ZSTD_decodeLiteralsBlock(const void* src, size_t srcSize, void* dst, size_t dstSize, void* buff2)
+static size_t local_ZSTD_decodeLiteralsBlock(const void* src, size_t srcSize, void* dst, size_t dstSize, void* buff2)
 {
     (void)src; (void)srcSize; (void)dst; (void)dstSize;
     return ZSTD_decodeLiteralsBlock((ZSTD_DCtx*)g_zdc, buff2, g_cSize);
 }
 
 extern size_t ZSTD_decodeSeqHeaders(ZSTD_DCtx* dctx, int* nbSeq, const void* src, size_t srcSize);
-size_t local_ZSTD_decodeSeqHeaders(const void* src, size_t srcSize, void* dst, size_t dstSize, void* buff2)
+static size_t local_ZSTD_decodeSeqHeaders(const void* src, size_t srcSize, void* dst, size_t dstSize, void* buff2)
 {
     int nbSeq;
     (void)src; (void)srcSize; (void)dst; (void)dstSize;
@@ -263,9 +263,9 @@ local_ZSTD_decompressStream(const void* src, size_t srcSize,
 }
 
 #ifndef ZSTD_DLL_IMPORT
-size_t local_ZSTD_compressContinue(const void* src, size_t srcSize,
-                                   void* dst, size_t dstCapacity,
-                                   void* buff2)
+static size_t local_ZSTD_compressContinue(const void* src, size_t srcSize,
+                                          void* dst, size_t dstCapacity,
+                                          void* buff2)
 {
     ZSTD_parameters p;
     ZSTD_frameParameters f = { 1 /* contentSizeHeader*/, 0, 0 };
@@ -276,9 +276,9 @@ size_t local_ZSTD_compressContinue(const void* src, size_t srcSize,
 }
 
 #define FIRST_BLOCK_SIZE 8
-size_t local_ZSTD_compressContinue_extDict(const void* src, size_t srcSize,
-                                           void* dst, size_t dstCapacity,
-                                           void* buff2)
+static size_t local_ZSTD_compressContinue_extDict(const void* src, size_t srcSize,
+                                                  void* dst, size_t dstCapacity,
+                                                  void* buff2)
 {
     BYTE firstBlockBuf[FIRST_BLOCK_SIZE];
 
@@ -305,9 +305,9 @@ size_t local_ZSTD_compressContinue_extDict(const void* src, size_t srcSize,
                             srcSize - FIRST_BLOCK_SIZE);
 }
 
-size_t local_ZSTD_decompressContinue(const void* src, size_t srcSize,
-                                           void* dst, size_t dstCapacity,
-                                           void* buff2)
+static size_t local_ZSTD_decompressContinue(const void* src, size_t srcSize,
+                                            void* dst, size_t dstCapacity,
+                                            void* buff2)
 {
     size_t regeneratedSize = 0;
     const BYTE* ip = (const BYTE*)buff2;
diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index 24e00e803..5616285b9 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -72,6 +72,8 @@ static UTIL_time_t g_displayClock = UTIL_TIME_INITIALIZER;
 *********************************************************/
 #undef MIN
 #undef MAX
+/* Declaring the function is it isn't unused */
+void FUZ_bug976(void);
 void FUZ_bug976(void)
 {   /* these constants shall not depend on MIN() macro */
     assert(ZSTD_HASHLOG_MAX < 31);
diff --git a/tests/paramgrill.c b/tests/paramgrill.c
index 68e1fbc4d..466a156d3 100644
--- a/tests/paramgrill.c
+++ b/tests/paramgrill.c
@@ -374,7 +374,7 @@ static U32 FUZ_rotl32(U32 x, U32 r)
     return ((x << r) | (x >> (32 - r)));
 }
 
-U32 FUZ_rand(U32* src)
+static U32 FUZ_rand(U32* src)
 {
     const U32 prime1 = 2654435761U;
     const U32 prime2 = 2246822519U;
@@ -1937,8 +1937,8 @@ static int benchSample(double compressibility, int cLevel)
 /* benchFiles() :
  * note: while this function takes a table of filenames,
  * in practice, only the first filename will be used */
-int benchFiles(const char** fileNamesTable, int nbFiles,
-               const char* dictFileName, int cLevel)
+static int benchFiles(const char** fileNamesTable, int nbFiles,
+                      const char* dictFileName, int cLevel)
 {
     buffers_t buf;
     contexts_t ctx;
diff --git a/tests/zstreamtest.c b/tests/zstreamtest.c
index 96136a625..4d32c19ea 100644
--- a/tests/zstreamtest.c
+++ b/tests/zstreamtest.c
@@ -84,7 +84,7 @@ static U64 g_clockTime = 0;
     @return : a 27 bits random value, from a 32-bits `seed`.
     `seed` is also modified */
 #define FUZ_rotl32(x,r) ((x << r) | (x >> (32 - r)))
-unsigned int FUZ_rand(unsigned int* seedPtr)
+static unsigned int FUZ_rand(unsigned int* seedPtr)
 {
     static const U32 prime2 = 2246822519U;
     U32 rand32 = *seedPtr;
@@ -2040,7 +2040,7 @@ _output_error:
 /*-*******************************************************
 *  Command line
 *********************************************************/
-int FUZ_usage(const char* programName)
+static int FUZ_usage(const char* programName)
 {
     DISPLAY( "Usage :\n");
     DISPLAY( "      %s [args]\n", programName);

From 109bd374741860753f8d6929f53264f7fc4b192e Mon Sep 17 00:00:00 2001
From: Nick Terrell 
Date: Thu, 27 Sep 2018 15:22:34 -0700
Subject: [PATCH 307/372] Include stddef.h for size_t

---
 lib/common/xxhash.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/lib/common/xxhash.c b/lib/common/xxhash.c
index 9d9c0e963..532b81619 100644
--- a/lib/common/xxhash.c
+++ b/lib/common/xxhash.c
@@ -98,6 +98,7 @@
 /* Modify the local functions below should you wish to use some other memory routines */
 /* for malloc(), free() */
 #include 
+#include      /* size_t */
 static void* XXH_malloc(size_t s) { return malloc(s); }
 static void  XXH_free  (void* p)  { free(p); }
 /* for memcpy() */

From aec1a3ec58a11a7dcff8d630a38f9c25db549a84 Mon Sep 17 00:00:00 2001
From: Nick Terrell 
Date: Thu, 27 Sep 2018 15:23:20 -0700
Subject: [PATCH 308/372] Change byte to value to avoid a GRUB typedef

---
 lib/decompress/zstd_decompress.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c
index e7e91b180..711b5b6d7 100644
--- a/lib/decompress/zstd_decompress.c
+++ b/lib/decompress/zstd_decompress.c
@@ -1788,10 +1788,10 @@ ZSTDLIB_API size_t ZSTD_insertBlock(ZSTD_DCtx* dctx, const void* blockStart, siz
 }
 
 
-static size_t ZSTD_generateNxBytes(void* dst, size_t dstCapacity, BYTE byte, size_t length)
+static size_t ZSTD_generateNxBytes(void* dst, size_t dstCapacity, BYTE value, size_t length)
 {
     if (length > dstCapacity) return ERROR(dstSize_tooSmall);
-    memset(dst, byte, length);
+    memset(dst, value, length);
     return length;
 }
 

From d8c73cd607bb56cc0e4ed77bd2ca59d58916fd60 Mon Sep 17 00:00:00 2001
From: Nick Terrell 
Date: Thu, 27 Sep 2018 15:49:31 -0700
Subject: [PATCH 309/372] Reset number of threads less often

---
 tests/zstreamtest.c | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/tests/zstreamtest.c b/tests/zstreamtest.c
index b56009323..105e63e7d 100644
--- a/tests/zstreamtest.c
+++ b/tests/zstreamtest.c
@@ -1864,8 +1864,9 @@ static int fuzzerTests_newAPI(U32 seed, U32 nbTests, unsigned startTest,
                     CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(zc, pledgedSrcSize) );
                 }
 
-                /* multi-threading parameters */
-                {   U32 const nbThreadsCandidate = (FUZ_rand(&lseed) & 4) + 1;
+                /* multi-threading parameters. Only adjust ocassionally for small tests. */
+                if (bigTests || (FUZ_rand(&lseed) & 0xF) == 0xF) {
+                    U32 const nbThreadsCandidate = (FUZ_rand(&lseed) & 4) + 1;
                     U32 const nbThreadsAdjusted = (windowLogMalus < nbThreadsCandidate) ? nbThreadsCandidate - windowLogMalus : 1;
                     U32 const nbThreads = MIN(nbThreadsAdjusted, nbThreadsMax);
                     DISPLAYLEVEL(5, "t%u: nbThreads : %u \n", testNb, nbThreads);

From a180ea07c4d945bf9400a70494bf458841a44437 Mon Sep 17 00:00:00 2001
From: Nick Terrell 
Date: Thu, 27 Sep 2018 16:06:02 -0700
Subject: [PATCH 310/372] Restore ZSTD_noCompressBlock() for clarity

---
 lib/compress/zstd_compress.c | 16 +++++++++++-----
 1 file changed, 11 insertions(+), 5 deletions(-)

diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index fcdf93f57..3eb5ceb21 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -1531,6 +1531,15 @@ static void ZSTD_reduceIndex (ZSTD_CCtx* zc, const U32 reducerValue)
 
 /* See doc/zstd_compression_format.md for detailed format description */
 
+static size_t ZSTD_noCompressBlock (void* dst, size_t dstCapacity, const void* src, size_t srcSize, U32 lastBlock)
+{
+    U32 const cBlockHeader24 = lastBlock + (((U32)bt_raw)<<1) + (U32)(srcSize << 3);
+    if (srcSize + ZSTD_blockHeaderSize > dstCapacity) return ERROR(dstSize_tooSmall);
+    MEM_writeLE24(dst, cBlockHeader24);
+    memcpy((BYTE*)dst + ZSTD_blockHeaderSize, src, srcSize);
+    return ZSTD_blockHeaderSize + srcSize;
+}
+
 static size_t ZSTD_noCompressLiterals (void* dst, size_t dstCapacity, const void* src, size_t srcSize)
 {
     BYTE* const ostart = (BYTE* const)dst;
@@ -2485,11 +2494,8 @@ static size_t ZSTD_compress_frameChunk (ZSTD_CCtx* cctx,
             if (ZSTD_isError(cSize)) return cSize;
 
             if (cSize == 0) {  /* block is not compressible */
-                U32 const cBlockHeader24 = lastBlock + (((U32)bt_raw)<<1) + (U32)(blockSize << 3);
-                if (blockSize + ZSTD_blockHeaderSize > dstCapacity) return ERROR(dstSize_tooSmall);
-                MEM_writeLE32(op, cBlockHeader24);   /* 4th byte will be overwritten */
-                memcpy(op + ZSTD_blockHeaderSize, ip, blockSize);
-                cSize = ZSTD_blockHeaderSize + blockSize;
+                cSize = ZSTD_noCompressBlock(op, dstCapacity, ip, blockSize, lastBlock);
+                if (ZSTD_isError(cSize)) return cSize;
             } else {
                 U32 const cBlockHeader24 = lastBlock + (((U32)bt_compressed)<<1) + (U32)(cSize << 3);
                 MEM_writeLE24(op, cBlockHeader24);

From 9b45db7fa6f2491aaecaa410031b4d6bced870f7 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Thu, 27 Sep 2018 16:49:08 -0700
Subject: [PATCH 311/372] minor refactoring of --list

trying to reduce recurrent patterns.
---
 programs/fileio.c  | 196 ++++++++++++++++++++-------------------------
 tests/playTests.sh |   9 ++-
 2 files changed, 97 insertions(+), 108 deletions(-)

diff --git a/programs/fileio.c b/programs/fileio.c
index 53f72aa72..c57792aa4 100644
--- a/programs/fileio.c
+++ b/programs/fileio.c
@@ -1988,22 +1988,19 @@ typedef struct {
     U32 nbFiles;
 } fileInfo_t;
 
-/** getFileInfo() :
- *  Reads information from file, stores in *info
- * @return : 0 if successful
- *           1 for frame analysis error
- *           2 for file not compressed with zstd
- *           3 for cases in which file could not be opened.
- */
-static int getFileInfo_fileConfirmed(fileInfo_t* info, const char* inFileName){
-    int detectError = 0;
-    FILE* const srcFile = FIO_openSrcFile(inFileName);
-    if (srcFile == NULL) {
-        DISPLAY("Error: could not open source file %s\n", inFileName);
-        return 3;
-    }
-    info->compressedSize = UTIL_getFileSize(inFileName);
+typedef enum { info_success=0, info_frame_error=1, info_not_zstd=2, info_file_error=3 } InfoError;
 
+#define EXIT_IF(c,n,...) {             \
+    if (c) {                           \
+        DISPLAYLEVEL(1, __VA_ARGS__);  \
+        DISPLAYLEVEL(1, " \n");        \
+        return n;                      \
+    }                                  \
+}
+
+static InfoError
+FIO_analyzeFrames(fileInfo_t* info, FILE* const srcFile)
+{
     /* begin analyzing frame */
     for ( ; ; ) {
         BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX];
@@ -2013,130 +2010,111 @@ static int getFileInfo_fileConfirmed(fileInfo_t* info, const char* inFileName){
               && (numBytesRead == 0)
               && (info->compressedSize > 0)
               && (info->compressedSize != UTIL_FILESIZE_UNKNOWN) ) {
-                break;
-            }
-            else if (feof(srcFile)) {
-                DISPLAY("Error: reached end of file with incomplete frame\n");
-                detectError = 2;
-                break;
-            }
-            else {
-                DISPLAY("Error: did not reach end of file but ran out of frames\n");
-                detectError = 1;
-                break;
+                return 0;  /* successful end of file */
             }
+            EXIT_IF(feof(srcFile), info_not_zstd, "Error: reached end of file with incomplete frame");
+            EXIT_IF(1, info_frame_error, "Error: did not reach end of file but ran out of frames");
         }
         {   U32 const magicNumber = MEM_readLE32(headerBuffer);
             /* Zstandard frame */
             if (magicNumber == ZSTD_MAGICNUMBER) {
                 ZSTD_frameHeader header;
                 U64 const frameContentSize = ZSTD_getFrameContentSize(headerBuffer, numBytesRead);
-                if (frameContentSize == ZSTD_CONTENTSIZE_ERROR || frameContentSize == ZSTD_CONTENTSIZE_UNKNOWN) {
+                if ( frameContentSize == ZSTD_CONTENTSIZE_ERROR
+                  || frameContentSize == ZSTD_CONTENTSIZE_UNKNOWN ) {
                     info->decompUnavailable = 1;
                 } else {
                     info->decompressedSize += frameContentSize;
                 }
-                if (ZSTD_getFrameHeader(&header, headerBuffer, numBytesRead) != 0) {
-                    DISPLAY("Error: could not decode frame header\n");
-                    detectError = 1;
-                    break;
-                }
+                EXIT_IF(ZSTD_getFrameHeader(&header, headerBuffer, numBytesRead) != 0,
+                        info_frame_error, "Error: could not decode frame header");
                 info->windowSize = header.windowSize;
                 /* move to the end of the frame header */
                 {   size_t const headerSize = ZSTD_frameHeaderSize(headerBuffer, numBytesRead);
-                    if (ZSTD_isError(headerSize)) {
-                        DISPLAY("Error: could not determine frame header size\n");
-                        detectError = 1;
-                        break;
-                    }
-                    {   int const ret = fseek(srcFile, ((long)headerSize)-((long)numBytesRead), SEEK_CUR);
-                        if (ret != 0) {
-                            DISPLAY("Error: could not move to end of frame header\n");
-                            detectError = 1;
-                            break;
-                }   }   }
+                    EXIT_IF(ZSTD_isError(headerSize), 1, "Error: could not determine frame header size");
+                    EXIT_IF(fseek(srcFile, ((long)headerSize)-((long)numBytesRead), SEEK_CUR) != 0,
+                            info_frame_error, "Error: could not move to end of frame header");
+                }
 
-                /* skip the rest of the blocks in the frame */
+                /* skip all blocks in the frame */
                 {   int lastBlock = 0;
                     do {
                         BYTE blockHeaderBuffer[3];
-                        size_t const readBytes = fread(blockHeaderBuffer, 1, 3, srcFile);
-                        if (readBytes != 3) {
-                            DISPLAY("There was a problem reading the block header\n");
-                            detectError = 1;
-                            break;
-                        }
+                        EXIT_IF(fread(blockHeaderBuffer, 1, 3, srcFile) != 3,
+                                info_frame_error, "Error while reading block header");
                         {   U32 const blockHeader = MEM_readLE24(blockHeaderBuffer);
                             U32 const blockTypeID = (blockHeader >> 1) & 3;
                             U32 const isRLE = (blockTypeID == 1);
                             U32 const isWrongBlock = (blockTypeID == 3);
                             long const blockSize = isRLE ? 1 : (long)(blockHeader >> 3);
-                            if (isWrongBlock) {
-                                DISPLAY("Error: unsupported block type \n");
-                                detectError = 1;
-                                break;
-                            }
+                            EXIT_IF(isWrongBlock, info_frame_error, "Error: unsupported block type");
                             lastBlock = blockHeader & 1;
-                            {   int const ret = fseek(srcFile, blockSize, SEEK_CUR);
-                                if (ret != 0) {
-                                    DISPLAY("Error: could not skip to end of block\n");
-                                    detectError = 1;
-                                    break;
-                        }   }   }
+                            EXIT_IF(fseek(srcFile, blockSize, SEEK_CUR) != 0,
+                                    info_frame_error, "Error: could not skip to end of block");
+                        }
                     } while (lastBlock != 1);
-
-                    if (detectError) break;
                 }
 
                 /* check if checksum is used */
                 {   BYTE const frameHeaderDescriptor = headerBuffer[4];
                     int const contentChecksumFlag = (frameHeaderDescriptor & (1 << 2)) >> 2;
                     if (contentChecksumFlag) {
-                        int const ret = fseek(srcFile, 4, SEEK_CUR);
                         info->usesCheck = 1;
-                        if (ret != 0) {
-                            DISPLAY("Error: could not skip past checksum\n");
-                            detectError = 1;
-                            break;
-                }   }   }
+                        EXIT_IF(fseek(srcFile, 4, SEEK_CUR) != 0,
+                                info_frame_error, "Error: could not skip past checksum");
+                }   }
                 info->numActualFrames++;
             }
             /* Skippable frame */
             else if ((magicNumber & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) {
                 U32 const frameSize = MEM_readLE32(headerBuffer + 4);
                 long const seek = (long)(8 + frameSize - numBytesRead);
-                int const ret = LONG_SEEK(srcFile, seek, SEEK_CUR);
-                if (ret != 0) {
-                    DISPLAY("Error: could not find end of skippable frame\n");
-                    detectError = 1;
-                    break;
-                }
+                EXIT_IF(LONG_SEEK(srcFile, seek, SEEK_CUR) != 0,
+                        info_frame_error, "Error: could not find end of skippable frame");
                 info->numSkippableFrames++;
             }
             /* unknown content */
             else {
-                detectError = 2;
-                break;
+                return info_not_zstd;
             }
-        }
-    }  /* end analyzing frame */
-    fclose(srcFile);
-    info->nbFiles = 1;
-    return detectError;
+        }  /* magic number analysis */
+    }  /* end analyzing frames */
+    return info_success;
 }
 
-static int getFileInfo(fileInfo_t* info, const char* srcFileName)
+
+static InfoError
+getFileInfo_fileConfirmed(fileInfo_t* info, const char* inFileName)
 {
-    int const isAFile = UTIL_isRegularFile(srcFileName);
-    if (!isAFile) {
-        DISPLAY("Error : %s is not a file", srcFileName);
-        return 3;
-    }
+    InfoError status;
+    FILE* const srcFile = FIO_openSrcFile(inFileName);
+    EXIT_IF(srcFile == NULL, info_file_error, "Error: could not open source file %s", inFileName);
+
+    info->compressedSize = UTIL_getFileSize(inFileName);
+    status = FIO_analyzeFrames(info, srcFile);
+
+    fclose(srcFile);
+    info->nbFiles = 1;
+    return status;
+}
+
+
+/** getFileInfo() :
+ *  Reads information from file, stores in *info
+ * @return : InfoError status
+ */
+static InfoError
+getFileInfo(fileInfo_t* info, const char* srcFileName)
+{
+    EXIT_IF(!UTIL_isRegularFile(srcFileName),
+            info_file_error, "Error : %s is not a file", srcFileName);
     return getFileInfo_fileConfirmed(info, srcFileName);
 }
 
 
-static void displayInfo(const char* inFileName, const fileInfo_t* info, int displayLevel){
+static void
+displayInfo(const char* inFileName, const fileInfo_t* info, int displayLevel)
+{
     unsigned const unit = info->compressedSize < (1 MB) ? (1 KB) : (1 MB);
     const char* const unitStr = info->compressedSize < (1 MB) ? "KB" : "MB";
     double const windowSizeUnit = (double)info->windowSize / unit;
@@ -2197,43 +2175,45 @@ static fileInfo_t FIO_addFInfo(fileInfo_t fi1, fileInfo_t fi2)
 static int FIO_listFile(fileInfo_t* total, const char* inFileName, int displayLevel){
     fileInfo_t info;
     memset(&info, 0, sizeof(info));
-    {   int const error = getFileInfo(&info, inFileName);
-        if (error == 1) {
+    {   InfoError const error = getFileInfo(&info, inFileName);
+        if (error == info_frame_error) {
             /* display error, but provide output */
-            DISPLAY("An error occurred while getting file info \n");
+            DISPLAYLEVEL(1, "Error while parsing %s \n", inFileName);
         }
-        else if (error == 2) {
+        else if (error == info_not_zstd) {
             DISPLAYOUT("File %s not compressed by zstd \n", inFileName);
             if (displayLevel > 2) DISPLAYOUT("\n");
             return 1;
         }
-        else if (error == 3) {
+        else if (error == info_file_error) {
             /* error occurred while opening the file */
             if (displayLevel > 2) DISPLAYOUT("\n");
             return 1;
         }
         displayInfo(inFileName, &info, displayLevel);
         *total = FIO_addFInfo(*total, info);
+        assert(error>=0 || error<=1);
         return error;
     }
 }
 
-int FIO_listMultipleFiles(unsigned numFiles, const char** filenameTable, int displayLevel){
-    unsigned u;
-    for (u=0; u 1 && displayLevel <= 2) {   /* display total */
             unsigned const unit = total.compressedSize < (1 MB) ? (1 KB) : (1 MB);
             const char* const unitStr = total.compressedSize < (1 MB) ? "KB" : "MB";
diff --git a/tests/playTests.sh b/tests/playTests.sh
index c7e066b84..8f16e4a7a 100755
--- a/tests/playTests.sh
+++ b/tests/playTests.sh
@@ -180,6 +180,8 @@ chmod 400 tmpro.zst
 $ZSTD -q tmpro && die "should have refused to overwrite read-only file"
 $ZSTD -q -f tmpro
 rm -f tmpro tmpro.zst
+
+
 $ECHO "test : file removal"
 $ZSTD -f --rm tmp
 test ! -f tmp  # tmp should no longer be present
@@ -196,9 +198,14 @@ $ECHO a | $ZSTD --rm > $INTOVOID   # --rm should remain silent
 rm tmp
 $ZSTD -f tmp && die "tmp not present : should have failed"
 test ! -f tmp.zst  # tmp.zst should not be created
+$ECHO "test : do not delete destination when source is not present"
+touch tmp    # create destination file
+$ZSTD -d -f tmp.zst && die "attempt to decompress a non existing file"
+! test -f tmp  # destination file should still be present (test disabled temporarily)
+rm tmp*
+
 
 $ECHO "test : compress multiple files"
-rm tmp*
 $ECHO hello > tmp1
 $ECHO world > tmp2
 $ZSTD tmp1 tmp2 -o "$INTOVOID"

From 73773c6b6a6f4dc96641d1951f3879bae1f7d553 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Thu, 27 Sep 2018 18:15:14 -0700
Subject: [PATCH 312/372] fixed legacy compilation tests

for some reason, these tests started failing recently on CircleCI
---
 lib/legacy/zstd_v01.c | 6 ++++++
 lib/legacy/zstd_v02.c | 6 ++++++
 lib/legacy/zstd_v03.c | 6 ++++++
 tests/legacy.c        | 5 +++--
 4 files changed, 21 insertions(+), 2 deletions(-)

diff --git a/lib/legacy/zstd_v01.c b/lib/legacy/zstd_v01.c
index 6303e1cfc..c007e7ceb 100644
--- a/lib/legacy/zstd_v01.c
+++ b/lib/legacy/zstd_v01.c
@@ -668,11 +668,17 @@ static size_t FSE_initDStream(FSE_DStream_t* bitD, const void* srcBuffer, size_t
         switch(srcSize)
         {
             case 7: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[6]) << (sizeof(size_t)*8 - 16);
+                    /* fallthrough */
             case 6: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[5]) << (sizeof(size_t)*8 - 24);
+                    /* fallthrough */
             case 5: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[4]) << (sizeof(size_t)*8 - 32);
+                    /* fallthrough */
             case 4: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[3]) << 24;
+                    /* fallthrough */
             case 3: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[2]) << 16;
+                    /* fallthrough */
             case 2: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[1]) <<  8;
+                    /* fallthrough */
             default:;
         }
         contain32 = ((const BYTE*)srcBuffer)[srcSize-1];
diff --git a/lib/legacy/zstd_v02.c b/lib/legacy/zstd_v02.c
index 8bc0eceed..c09ef8cff 100644
--- a/lib/legacy/zstd_v02.c
+++ b/lib/legacy/zstd_v02.c
@@ -399,11 +399,17 @@ MEM_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, si
         switch(srcSize)
         {
             case 7: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[6]) << (sizeof(size_t)*8 - 16);
+                    /* fallthrough */
             case 6: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[5]) << (sizeof(size_t)*8 - 24);
+                    /* fallthrough */
             case 5: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[4]) << (sizeof(size_t)*8 - 32);
+                    /* fallthrough */
             case 4: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[3]) << 24;
+                    /* fallthrough */
             case 3: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[2]) << 16;
+                    /* fallthrough */
             case 2: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[1]) <<  8;
+                    /* fallthrough */
             default:;
         }
         contain32 = ((const BYTE*)srcBuffer)[srcSize-1];
diff --git a/lib/legacy/zstd_v03.c b/lib/legacy/zstd_v03.c
index 54445af57..0c4cdf688 100644
--- a/lib/legacy/zstd_v03.c
+++ b/lib/legacy/zstd_v03.c
@@ -402,11 +402,17 @@ MEM_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, si
         switch(srcSize)
         {
             case 7: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[6]) << (sizeof(size_t)*8 - 16);
+                    /* fallthrough */
             case 6: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[5]) << (sizeof(size_t)*8 - 24);
+                    /* fallthrough */
             case 5: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[4]) << (sizeof(size_t)*8 - 32);
+                    /* fallthrough */
             case 4: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[3]) << 24;
+                    /* fallthrough */
             case 3: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[2]) << 16;
+                    /* fallthrough */
             case 2: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[1]) <<  8;
+                    /* fallthrough */
             default:;
         }
         contain32 = ((const BYTE*)srcBuffer)[srcSize-1];
diff --git a/tests/legacy.c b/tests/legacy.c
index 847e1d25e..e1cf82f2f 100644
--- a/tests/legacy.c
+++ b/tests/legacy.c
@@ -36,7 +36,7 @@ size_t const COMPRESSED_SIZE = 917;
 const char* const EXPECTED; /* content is at end of file */
 
 
-int testSimpleAPI(void)
+static int testSimpleAPI(void)
 {
     size_t const size = strlen(EXPECTED);
     char* const output = malloc(size);
@@ -71,7 +71,8 @@ int testSimpleAPI(void)
     return 0;
 }
 
-int testStreamingAPI(void)
+
+static int testStreamingAPI(void)
 {
     size_t const outBuffSize = ZSTD_DStreamOutSize();
     char* const outBuff = malloc(outBuffSize);

From ff36513556f6484d6570291d90fb25eb4c4f2751 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Thu, 27 Sep 2018 18:24:41 -0700
Subject: [PATCH 313/372] fixed longmatch test too

---
 tests/longmatch.c | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/tests/longmatch.c b/tests/longmatch.c
index ed3861571..1271e9ab1 100644
--- a/tests/longmatch.c
+++ b/tests/longmatch.c
@@ -17,25 +17,25 @@
 #define ZSTD_STATIC_LINKING_ONLY
 #include "zstd.h"
 
-int compress(ZSTD_CStream *ctx, ZSTD_outBuffer out, const void *data, size_t size) {
+static int
+compress(ZSTD_CStream *ctx, ZSTD_outBuffer out, const void *data, size_t size)
+{
   ZSTD_inBuffer in = { data, size, 0 };
   while (in.pos < in.size) {
     ZSTD_outBuffer tmp = out;
     const size_t rc = ZSTD_compressStream(ctx, &tmp, &in);
-    if (ZSTD_isError(rc)) {
-      return 1;
-    }
+    if (ZSTD_isError(rc)) return 1;
   }
-  {
-    ZSTD_outBuffer tmp = out;
+  { ZSTD_outBuffer tmp = out;
     const size_t rc = ZSTD_flushStream(ctx, &tmp);
     if (rc != 0) { return 1; }
   }
   return 0;
 }
 
-int main(int argc, const char** argv) {
-  ZSTD_CStream *ctx;
+int main(int argc, const char** argv)
+{
+  ZSTD_CStream* ctx;
   ZSTD_parameters params;
   size_t rc;
   unsigned windowLog;

From ef1272737bd2f33044497297715a2bb48d8a1ee5 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Thu, 27 Sep 2018 18:29:15 -0700
Subject: [PATCH 314/372] fixed minor Visual conversion warnings

---
 programs/fileio.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/programs/fileio.c b/programs/fileio.c
index c57792aa4..6c9386c7c 100644
--- a/programs/fileio.c
+++ b/programs/fileio.c
@@ -2010,7 +2010,7 @@ FIO_analyzeFrames(fileInfo_t* info, FILE* const srcFile)
               && (numBytesRead == 0)
               && (info->compressedSize > 0)
               && (info->compressedSize != UTIL_FILESIZE_UNKNOWN) ) {
-                return 0;  /* successful end of file */
+                return info_success;
             }
             EXIT_IF(feof(srcFile), info_not_zstd, "Error: reached end of file with incomplete frame");
             EXIT_IF(1, info_frame_error, "Error: did not reach end of file but ran out of frames");
@@ -2031,7 +2031,7 @@ FIO_analyzeFrames(fileInfo_t* info, FILE* const srcFile)
                 info->windowSize = header.windowSize;
                 /* move to the end of the frame header */
                 {   size_t const headerSize = ZSTD_frameHeaderSize(headerBuffer, numBytesRead);
-                    EXIT_IF(ZSTD_isError(headerSize), 1, "Error: could not determine frame header size");
+                    EXIT_IF(ZSTD_isError(headerSize), info_frame_error, "Error: could not determine frame header size");
                     EXIT_IF(fseek(srcFile, ((long)headerSize)-((long)numBytesRead), SEEK_CUR) != 0,
                             info_frame_error, "Error: could not move to end of frame header");
                 }

From d987ab5983c5bc1660f53d6026a63569675e18e7 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Fri, 28 Sep 2018 09:34:16 -0700
Subject: [PATCH 315/372] fixed unreachable section warning on Visual

---
 programs/fileio.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/programs/fileio.c b/programs/fileio.c
index e780d481d..2c4e9f6e8 100644
--- a/programs/fileio.c
+++ b/programs/fileio.c
@@ -2010,7 +2010,7 @@ FIO_analyzeFrames(fileInfo_t* info, FILE* const srcFile)
               && (numBytesRead == 0)
               && (info->compressedSize > 0)
               && (info->compressedSize != UTIL_FILESIZE_UNKNOWN) ) {
-                return info_success;
+                break;  /* correct end of file => success */
             }
             EXIT_IF(feof(srcFile), info_not_zstd, "Error: reached end of file with incomplete frame");
             EXIT_IF(1, info_frame_error, "Error: did not reach end of file but ran out of frames");

From 146049a1ea7908342ed99e91275739a4e730d8c5 Mon Sep 17 00:00:00 2001
From: Nick Terrell 
Date: Fri, 28 Sep 2018 12:09:14 -0700
Subject: [PATCH 316/372] [zstreamtest] Add failing test case

---
 tests/zstreamtest.c | 39 +++++++++++++++++++++++++++++++++++++++
 1 file changed, 39 insertions(+)

diff --git a/tests/zstreamtest.c b/tests/zstreamtest.c
index a7af89eba..d78f00683 100644
--- a/tests/zstreamtest.c
+++ b/tests/zstreamtest.c
@@ -1073,6 +1073,45 @@ static int basicUnitTests(U32 seed, double compressibility)
     }
     DISPLAYLEVEL(3, "OK \n");
 
+    DISPLAYLEVEL(3, "test%3i : dictionary + small blocks + reusing tables checks offset table validity: ", testNb++);
+    {   ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(
+            dictionary.start, dictionary.filled,
+            ZSTD_dlm_byRef, ZSTD_dct_fullDict,
+            ZSTD_getCParams(3, 0, dictionary.filled),
+            ZSTD_defaultCMem);
+        ZSTD_outBuffer out = {compressedBuffer, compressedBufferSize, 0};
+        int remainingInput = 256 * 1024;
+
+        ZSTD_CCtx_reset(zc);
+        CHECK_Z(ZSTD_CCtx_resetParameters(zc));
+        CHECK_Z(ZSTD_CCtx_refCDict(zc, cdict));
+        CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_p_checksumFlag, 1));
+        /* Write a bunch of 6 byte blocks */
+        while (remainingInput > 0) {
+          const size_t kSmallBlockSize = 6;
+          char testBuffer[kSmallBlockSize] = "\xAA\xAA\xAA\xAA\xAA\xAA";
+          const size_t outStart = out.pos;
+          ZSTD_inBuffer in = {testBuffer, kSmallBlockSize, 0};
+
+          CHECK_Z(ZSTD_compress_generic(zc, &out, &in, ZSTD_e_flush));
+          CHECK(in.pos != in.size, "input not fully consumed");
+          remainingInput -= kSmallBlockSize;
+        }
+        /* Write several very long offset matches into the dictionary */
+        for (int offset = 1024; offset >= 0; offset -= 128) {
+          size_t start = out.pos;
+          ZSTD_inBuffer in = {dictionary.start + offset, 128, 0};
+          ZSTD_EndDirective flush = offset > 0 ? ZSTD_e_continue : ZSTD_e_end;
+          CHECK_Z(ZSTD_compress_generic(zc, &out, &in, flush));
+          CHECK(in.pos != in.size, "input not fully consumed");
+        }
+        /* Ensure decompression works */
+        CHECK_Z(ZSTD_decompress_usingDict(zd, decodedBuffer, CNBufferSize, out.dst, out.pos, dictionary.start, dictionary.filled));
+
+        ZSTD_freeCDict(cdict);
+    }
+    DISPLAYLEVEL(3, "OK \n");
+
 _end:
     FUZ_freeDictionary(dictionary);
     ZSTD_freeCStream(zc);

From 6391cd103035d594ed31d356418dcf8d0ceef6b2 Mon Sep 17 00:00:00 2001
From: Nick Terrell 
Date: Fri, 28 Sep 2018 12:09:28 -0700
Subject: [PATCH 317/372] [zstd] Fix newly added  test case

---
 lib/compress/zstd_compress.c | 42 +++++++++++++++++++-----------------
 1 file changed, 22 insertions(+), 20 deletions(-)

diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index 3eb5ceb21..5e0fb371c 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -2349,13 +2349,15 @@ static size_t ZSTD_compressBlock_internal(ZSTD_CCtx* zc,
                                         const void* src, size_t srcSize)
 {
     ZSTD_matchState_t* const ms = &zc->blockState.matchState;
+    size_t cSize;
     DEBUGLOG(5, "ZSTD_compressBlock_internal (dstCapacity=%zu, dictLimit=%u, nextToUpdate=%u)",
                 dstCapacity, ms->window.dictLimit, ms->nextToUpdate);
     assert(srcSize <= ZSTD_BLOCKSIZE_MAX);
 
     if (srcSize < MIN_CBLOCK_SIZE+ZSTD_blockHeaderSize+1) {
         ZSTD_ldm_skipSequences(&zc->externSeqStore, srcSize, zc->appliedParams.cParams.searchLength);
-        return 0;   /* don't even attempt compression below a certain srcSize */
+        cSize = 0;
+        goto out;  /* don't even attempt compression below a certain srcSize */
     }
     ZSTD_resetSeqStore(&(zc->seqStore));
     ms->opt.symbolCosts = &zc->blockState.prevCBlock->entropy;   /* required for optimal parser to read stats from dictionary */
@@ -2417,27 +2419,27 @@ static size_t ZSTD_compressBlock_internal(ZSTD_CCtx* zc,
     }   }
 
     /* encode sequences and literals */
-    {   size_t const cSize = ZSTD_compressSequences(&zc->seqStore,
-                                &zc->blockState.prevCBlock->entropy, &zc->blockState.nextCBlock->entropy,
-                                &zc->appliedParams,
-                                dst, dstCapacity,
-                                srcSize, zc->entropyWorkspace, zc->bmi2);
-        if (!ZSTD_isError(cSize) && cSize != 0) {
-            /* confirm repcodes and entropy tables */
-            ZSTD_compressedBlockState_t* const tmp = zc->blockState.prevCBlock;
-            zc->blockState.prevCBlock = zc->blockState.nextCBlock;
-            zc->blockState.nextCBlock = tmp;
-        }
+    cSize = ZSTD_compressSequences(&zc->seqStore,
+            &zc->blockState.prevCBlock->entropy, &zc->blockState.nextCBlock->entropy,
+            &zc->appliedParams,
+            dst, dstCapacity,
+            srcSize, zc->entropyWorkspace, zc->bmi2);
 
-        /* We check that dictionaries have offset codes available for the first
-         * block. After the first block, the offcode table might not have large
-         * enough codes to represent the offsets in the data.
-         */
-        if (zc->blockState.prevCBlock->entropy.fse.offcode_repeatMode == FSE_repeat_valid)
-            zc->blockState.prevCBlock->entropy.fse.offcode_repeatMode = FSE_repeat_check;
-
-        return cSize;
+out:
+    if (!ZSTD_isError(cSize) && cSize != 0) {
+        /* confirm repcodes and entropy tables when emitting a compressed block */
+        ZSTD_compressedBlockState_t* const tmp = zc->blockState.prevCBlock;
+        zc->blockState.prevCBlock = zc->blockState.nextCBlock;
+        zc->blockState.nextCBlock = tmp;
     }
+    /* We check that dictionaries have offset codes available for the first
+     * block. After the first block, the offcode table might not have large
+     * enough codes to represent the offsets in the data.
+     */
+    if (zc->blockState.prevCBlock->entropy.fse.offcode_repeatMode == FSE_repeat_valid)
+        zc->blockState.prevCBlock->entropy.fse.offcode_repeatMode = FSE_repeat_check;
+
+    return cSize;
 }
 
 

From 0e7a7f1def5db7b627130d38f25caae507f66de6 Mon Sep 17 00:00:00 2001
From: Nick Terrell 
Date: Fri, 28 Sep 2018 12:14:24 -0700
Subject: [PATCH 318/372] Fix warnings

---
 tests/zstreamtest.c | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/tests/zstreamtest.c b/tests/zstreamtest.c
index d78f00683..d7c8567d4 100644
--- a/tests/zstreamtest.c
+++ b/tests/zstreamtest.c
@@ -1088,9 +1088,8 @@ static int basicUnitTests(U32 seed, double compressibility)
         CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_p_checksumFlag, 1));
         /* Write a bunch of 6 byte blocks */
         while (remainingInput > 0) {
-          const size_t kSmallBlockSize = 6;
-          char testBuffer[kSmallBlockSize] = "\xAA\xAA\xAA\xAA\xAA\xAA";
-          const size_t outStart = out.pos;
+          char testBuffer[6] = "\xAA\xAA\xAA\xAA\xAA\xAA";
+          const size_t kSmallBlockSize = sizeof(testBuffer);
           ZSTD_inBuffer in = {testBuffer, kSmallBlockSize, 0};
 
           CHECK_Z(ZSTD_compress_generic(zc, &out, &in, ZSTD_e_flush));
@@ -1099,7 +1098,6 @@ static int basicUnitTests(U32 seed, double compressibility)
         }
         /* Write several very long offset matches into the dictionary */
         for (int offset = 1024; offset >= 0; offset -= 128) {
-          size_t start = out.pos;
           ZSTD_inBuffer in = {dictionary.start + offset, 128, 0};
           ZSTD_EndDirective flush = offset > 0 ? ZSTD_e_continue : ZSTD_e_end;
           CHECK_Z(ZSTD_compress_generic(zc, &out, &in, flush));

From eb4423e7edfeb1074e32c7619dbee2c7f2f6d93f Mon Sep 17 00:00:00 2001
From: Nick Terrell 
Date: Fri, 28 Sep 2018 14:24:38 -0700
Subject: [PATCH 319/372] Fix another warning

---
 tests/zstreamtest.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tests/zstreamtest.c b/tests/zstreamtest.c
index d7c8567d4..f47451a3c 100644
--- a/tests/zstreamtest.c
+++ b/tests/zstreamtest.c
@@ -1081,6 +1081,7 @@ static int basicUnitTests(U32 seed, double compressibility)
             ZSTD_defaultCMem);
         ZSTD_outBuffer out = {compressedBuffer, compressedBufferSize, 0};
         int remainingInput = 256 * 1024;
+        int offset;
 
         ZSTD_CCtx_reset(zc);
         CHECK_Z(ZSTD_CCtx_resetParameters(zc));
@@ -1097,7 +1098,7 @@ static int basicUnitTests(U32 seed, double compressibility)
           remainingInput -= kSmallBlockSize;
         }
         /* Write several very long offset matches into the dictionary */
-        for (int offset = 1024; offset >= 0; offset -= 128) {
+        for (offset = 1024; offset >= 0; offset -= 128) {
           ZSTD_inBuffer in = {dictionary.start + offset, 128, 0};
           ZSTD_EndDirective flush = offset > 0 ? ZSTD_e_continue : ZSTD_e_end;
           CHECK_Z(ZSTD_compress_generic(zc, &out, &in, flush));

From 43146d8a16fc287bb96394410a3bb6d4fa8aeb7a Mon Sep 17 00:00:00 2001
From: Nick Terrell 
Date: Fri, 28 Sep 2018 14:59:40 -0700
Subject: [PATCH 320/372] Add -Werror to *build rules

---
 Makefile | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/Makefile b/Makefile
index 4011250c5..53e993d1b 100644
--- a/Makefile
+++ b/Makefile
@@ -340,23 +340,23 @@ cmakebuild:
 
 c90build: clean
 	$(CC) -v
-	CFLAGS="-std=c90" $(MAKE) allmost  # will fail, due to missing support for `long long`
+	CFLAGS="-std=c90 -Werror" $(MAKE) allmost  # will fail, due to missing support for `long long`
 
 gnu90build: clean
 	$(CC) -v
-	CFLAGS="-std=gnu90" $(MAKE) allmost
+	CFLAGS="-std=gnu90 -Werror" $(MAKE) allmost
 
 c99build: clean
 	$(CC) -v
-	CFLAGS="-std=c99" $(MAKE) allmost
+	CFLAGS="-std=c99 -Werror" $(MAKE) allmost
 
 gnu99build: clean
 	$(CC) -v
-	CFLAGS="-std=gnu99" $(MAKE) allmost
+	CFLAGS="-std=gnu99 -Werror" $(MAKE) allmost
 
 c11build: clean
 	$(CC) -v
-	CFLAGS="-std=c11" $(MAKE) allmost
+	CFLAGS="-std=c11 -Werror" $(MAKE) allmost
 
 bmix64build: clean
 	$(CC) -v

From 09231dad4d7ba1414c639ccfdd43a7355697a905 Mon Sep 17 00:00:00 2001
From: Nick Terrell 
Date: Fri, 28 Sep 2018 15:08:15 -0700
Subject: [PATCH 321/372] [util] Fix lstat feature test macro

---
 programs/util.h | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/programs/util.h b/programs/util.h
index e8288b8fe..d6184fac8 100644
--- a/programs/util.h
+++ b/programs/util.h
@@ -324,6 +324,7 @@ UTIL_STATIC U32 UTIL_isDirectory(const char* infilename)
 UTIL_STATIC U32 UTIL_isLink(const char* infilename)
 {
 /* macro guards, as defined in : https://linux.die.net/man/2/lstat */
+#ifndef __STRICT_ANSI__
 #if defined(_BSD_SOURCE) \
     || (defined(_XOPEN_SOURCE) && (_XOPEN_SOURCE >= 500)) \
     || (defined(_XOPEN_SOURCE) && defined(_XOPEN_SOURCE_EXTENDED)) \
@@ -333,6 +334,7 @@ UTIL_STATIC U32 UTIL_isLink(const char* infilename)
     stat_t statbuf;
     r = lstat(infilename, &statbuf);
     if (!r && S_ISLNK(statbuf.st_mode)) return 1;
+#endif
 #endif
     (void)infilename;
     return 0;

From 5aa9a1dd2d5140a2a8b06c447336a723ba85f578 Mon Sep 17 00:00:00 2001
From: Nick Terrell 
Date: Fri, 28 Sep 2018 15:28:34 -0700
Subject: [PATCH 322/372] Fix minigzip in std=c99 mode

---
 zlibWrapper/examples/minigzip.c | 18 ++----------------
 1 file changed, 2 insertions(+), 16 deletions(-)

diff --git a/zlibWrapper/examples/minigzip.c b/zlibWrapper/examples/minigzip.c
index 521d04711..f67be0956 100644
--- a/zlibWrapper/examples/minigzip.c
+++ b/zlibWrapper/examples/minigzip.c
@@ -18,6 +18,8 @@
 
 /* @(#) $Id$ */
 
+#define _POSIX_SOURCE /* fileno */
+
 #include "zstd_zlibwrapper.h"
 #include 
 
@@ -470,12 +472,8 @@ void file_compress(file, mode)
         exit(1);
     }
 
-#if !defined(NO_snprintf) && !defined(NO_vsnprintf)
-    snprintf(outfile, sizeof(outfile), "%s%s", file, GZ_SUFFIX);
-#else
     strcpy(outfile, file);
     strcat(outfile, GZ_SUFFIX);
-#endif
 
     in = fopen(file, "rb");
     if (in == NULL) {
@@ -510,11 +508,7 @@ void file_uncompress(file)
         exit(1);
     }
 
-#if !defined(NO_snprintf) && !defined(NO_vsnprintf)
-    snprintf(buf, sizeof(buf), "%s", file);
-#else
     strcpy(buf, file);
-#endif
 
     if (len > SUFFIX_LEN && strcmp(file+len-SUFFIX_LEN, GZ_SUFFIX) == 0) {
         infile = file;
@@ -523,11 +517,7 @@ void file_uncompress(file)
     } else {
         outfile = file;
         infile = buf;
-#if !defined(NO_snprintf) && !defined(NO_vsnprintf)
-        snprintf(buf + len, sizeof(buf) - len, "%s", GZ_SUFFIX);
-#else
         strcat(infile, GZ_SUFFIX);
-#endif
     }
     in = gzopen(infile, "rb");
     if (in == NULL) {
@@ -565,11 +555,7 @@ int main(argc, argv)
     gzFile file;
     char *bname, outmode[20];
 
-#if !defined(NO_snprintf) && !defined(NO_vsnprintf)
-    snprintf(outmode, sizeof(outmode), "%s", "wb6 ");
-#else
     strcpy(outmode, "wb6 ");
-#endif
 
     prog = argv[0];
     bname = strrchr(argv[0], '/');

From 05c0a072b718d33e451bf84f4b600a9c1e1ba6aa Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Fri, 28 Sep 2018 15:57:35 -0700
Subject: [PATCH 323/372] minor improvement in the multi-format suffix
 selection

---
 programs/fileio.c | 29 ++++++++++++++++++-----------
 1 file changed, 18 insertions(+), 11 deletions(-)

diff --git a/programs/fileio.c b/programs/fileio.c
index 2c4e9f6e8..be0037c24 100644
--- a/programs/fileio.c
+++ b/programs/fileio.c
@@ -1914,9 +1914,9 @@ int FIO_decompressMultipleFilenames(const char** srcNamesTable, unsigned nbFiles
             EXM_THROW(72, "Write error : cannot properly close output file");
     } else {
         size_t suffixSize;
-        size_t dfnSize = FNSPACE;
+        size_t dfnbCapacity = FNSPACE;
         unsigned u;
-        char* dstFileName = (char*)malloc(FNSPACE);
+        char* dstFileName = (char*)malloc(dfnbCapacity);
         if (dstFileName==NULL)
             EXM_THROW(73, "not enough memory for dstFileName");
         for (u=0; u
Date: Fri, 28 Sep 2018 16:04:00 -0700
Subject: [PATCH 324/372] changed macro name

from EXIT_IF() to RETURN_IF()

EXIT could be misunderstood as exit(), which terminates program execution.
But the macro only leaves the function, not the program.
---
 programs/fileio.c | 32 +++++++++++++++++---------------
 1 file changed, 17 insertions(+), 15 deletions(-)

diff --git a/programs/fileio.c b/programs/fileio.c
index be0037c24..12d24603f 100644
--- a/programs/fileio.c
+++ b/programs/fileio.c
@@ -1997,7 +1997,7 @@ typedef struct {
 
 typedef enum { info_success=0, info_frame_error=1, info_not_zstd=2, info_file_error=3 } InfoError;
 
-#define EXIT_IF(c,n,...) {             \
+#define ERROR_IF(c,n,...) {             \
     if (c) {                           \
         DISPLAYLEVEL(1, __VA_ARGS__);  \
         DISPLAYLEVEL(1, " \n");        \
@@ -2019,8 +2019,8 @@ FIO_analyzeFrames(fileInfo_t* info, FILE* const srcFile)
               && (info->compressedSize != UTIL_FILESIZE_UNKNOWN) ) {
                 break;  /* correct end of file => success */
             }
-            EXIT_IF(feof(srcFile), info_not_zstd, "Error: reached end of file with incomplete frame");
-            EXIT_IF(1, info_frame_error, "Error: did not reach end of file but ran out of frames");
+            ERROR_IF(feof(srcFile), info_not_zstd, "Error: reached end of file with incomplete frame");
+            ERROR_IF(1, info_frame_error, "Error: did not reach end of file but ran out of frames");
         }
         {   U32 const magicNumber = MEM_readLE32(headerBuffer);
             /* Zstandard frame */
@@ -2033,13 +2033,13 @@ FIO_analyzeFrames(fileInfo_t* info, FILE* const srcFile)
                 } else {
                     info->decompressedSize += frameContentSize;
                 }
-                EXIT_IF(ZSTD_getFrameHeader(&header, headerBuffer, numBytesRead) != 0,
+                ERROR_IF(ZSTD_getFrameHeader(&header, headerBuffer, numBytesRead) != 0,
                         info_frame_error, "Error: could not decode frame header");
                 info->windowSize = header.windowSize;
                 /* move to the end of the frame header */
                 {   size_t const headerSize = ZSTD_frameHeaderSize(headerBuffer, numBytesRead);
-                    EXIT_IF(ZSTD_isError(headerSize), info_frame_error, "Error: could not determine frame header size");
-                    EXIT_IF(fseek(srcFile, ((long)headerSize)-((long)numBytesRead), SEEK_CUR) != 0,
+                    ERROR_IF(ZSTD_isError(headerSize), info_frame_error, "Error: could not determine frame header size");
+                    ERROR_IF(fseek(srcFile, ((long)headerSize)-((long)numBytesRead), SEEK_CUR) != 0,
                             info_frame_error, "Error: could not move to end of frame header");
                 }
 
@@ -2047,16 +2047,16 @@ FIO_analyzeFrames(fileInfo_t* info, FILE* const srcFile)
                 {   int lastBlock = 0;
                     do {
                         BYTE blockHeaderBuffer[3];
-                        EXIT_IF(fread(blockHeaderBuffer, 1, 3, srcFile) != 3,
+                        ERROR_IF(fread(blockHeaderBuffer, 1, 3, srcFile) != 3,
                                 info_frame_error, "Error while reading block header");
                         {   U32 const blockHeader = MEM_readLE24(blockHeaderBuffer);
                             U32 const blockTypeID = (blockHeader >> 1) & 3;
                             U32 const isRLE = (blockTypeID == 1);
                             U32 const isWrongBlock = (blockTypeID == 3);
                             long const blockSize = isRLE ? 1 : (long)(blockHeader >> 3);
-                            EXIT_IF(isWrongBlock, info_frame_error, "Error: unsupported block type");
+                            ERROR_IF(isWrongBlock, info_frame_error, "Error: unsupported block type");
                             lastBlock = blockHeader & 1;
-                            EXIT_IF(fseek(srcFile, blockSize, SEEK_CUR) != 0,
+                            ERROR_IF(fseek(srcFile, blockSize, SEEK_CUR) != 0,
                                     info_frame_error, "Error: could not skip to end of block");
                         }
                     } while (lastBlock != 1);
@@ -2067,7 +2067,7 @@ FIO_analyzeFrames(fileInfo_t* info, FILE* const srcFile)
                     int const contentChecksumFlag = (frameHeaderDescriptor & (1 << 2)) >> 2;
                     if (contentChecksumFlag) {
                         info->usesCheck = 1;
-                        EXIT_IF(fseek(srcFile, 4, SEEK_CUR) != 0,
+                        ERROR_IF(fseek(srcFile, 4, SEEK_CUR) != 0,
                                 info_frame_error, "Error: could not skip past checksum");
                 }   }
                 info->numActualFrames++;
@@ -2076,7 +2076,7 @@ FIO_analyzeFrames(fileInfo_t* info, FILE* const srcFile)
             else if ((magicNumber & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) {
                 U32 const frameSize = MEM_readLE32(headerBuffer + 4);
                 long const seek = (long)(8 + frameSize - numBytesRead);
-                EXIT_IF(LONG_SEEK(srcFile, seek, SEEK_CUR) != 0,
+                ERROR_IF(LONG_SEEK(srcFile, seek, SEEK_CUR) != 0,
                         info_frame_error, "Error: could not find end of skippable frame");
                 info->numSkippableFrames++;
             }
@@ -2095,7 +2095,7 @@ getFileInfo_fileConfirmed(fileInfo_t* info, const char* inFileName)
 {
     InfoError status;
     FILE* const srcFile = FIO_openSrcFile(inFileName);
-    EXIT_IF(srcFile == NULL, info_file_error, "Error: could not open source file %s", inFileName);
+    ERROR_IF(srcFile == NULL, info_file_error, "Error: could not open source file %s", inFileName);
 
     info->compressedSize = UTIL_getFileSize(inFileName);
     status = FIO_analyzeFrames(info, srcFile);
@@ -2113,7 +2113,7 @@ getFileInfo_fileConfirmed(fileInfo_t* info, const char* inFileName)
 static InfoError
 getFileInfo(fileInfo_t* info, const char* srcFileName)
 {
-    EXIT_IF(!UTIL_isRegularFile(srcFileName),
+    ERROR_IF(!UTIL_isRegularFile(srcFileName),
             info_file_error, "Error : %s is not a file", srcFileName);
     return getFileInfo_fileConfirmed(info, srcFileName);
 }
@@ -2179,7 +2179,9 @@ static fileInfo_t FIO_addFInfo(fileInfo_t fi1, fileInfo_t fi2)
     return total;
 }
 
-static int FIO_listFile(fileInfo_t* total, const char* inFileName, int displayLevel){
+static int
+FIO_listFile(fileInfo_t* total, const char* inFileName, int displayLevel)
+{
     fileInfo_t info;
     memset(&info, 0, sizeof(info));
     {   InfoError const error = getFileInfo(&info, inFileName);
@@ -2209,7 +2211,7 @@ int FIO_listMultipleFiles(unsigned numFiles, const char** filenameTable, int dis
     /* ensure no specified input is stdin (needs fseek() capability) */
     {   unsigned u;
         for (u=0; u
Date: Thu, 23 Aug 2018 11:32:32 -0700
Subject: [PATCH 325/372] Add ZSTD_compressionParameters to ZSTD_matchState_t

---
 lib/compress/zstd_compress.c          | 3 +++
 lib/compress/zstd_compress_internal.h | 1 +
 2 files changed, 4 insertions(+)

diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index 3eb5ceb21..5c508568f 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -1102,6 +1102,8 @@ ZSTD_reset_matchState(ZSTD_matchState_t* ms,
     ms->hashTable3 = ms->chainTable + chainSize;
     ptr = ms->hashTable3 + h3Size;
 
+    ms->cParams = *cParams;
+
     assert(((size_t)ptr & 3) == 0);
     return ptr;
 }
@@ -1376,6 +1378,7 @@ static size_t ZSTD_resetCCtx_usingCDict(ZSTD_CCtx* cctx,
             dstMatchState->nextToUpdate = srcMatchState->nextToUpdate;
             dstMatchState->nextToUpdate3= srcMatchState->nextToUpdate3;
             dstMatchState->loadedDictEnd= srcMatchState->loadedDictEnd;
+            dstMatchState->cParams      = srcMatchState->cParams;
         }
     }
 
diff --git a/lib/compress/zstd_compress_internal.h b/lib/compress/zstd_compress_internal.h
index 1c95b7de9..1c45d1b07 100644
--- a/lib/compress/zstd_compress_internal.h
+++ b/lib/compress/zstd_compress_internal.h
@@ -140,6 +140,7 @@ struct ZSTD_matchState_t {
     U32* chainTable;
     optState_t opt;         /* optimal parser state */
     const ZSTD_matchState_t *dictMatchState;
+    ZSTD_compressionParameters cParams;
 };
 
 typedef struct {

From 4e3ecee9edd87c9b8ab5f1ce809582c4637cd489 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" 
Date: Thu, 23 Aug 2018 11:33:08 -0700
Subject: [PATCH 326/372] Remove cParams from CDict

---
 lib/compress/zstd_compress.c | 22 +++++++++++-----------
 1 file changed, 11 insertions(+), 11 deletions(-)

diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index 5c508568f..ae3626b6d 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -46,7 +46,6 @@ struct ZSTD_CDict_s {
     size_t workspaceSize;
     ZSTD_matchState_t matchState;
     ZSTD_compressedBlockState_t cBlockState;
-    ZSTD_compressionParameters cParams;
     ZSTD_customMem customMem;
     U32 dictID;
 };  /* typedef'd to ZSTD_CDict within "zstd.h" */
@@ -1307,14 +1306,15 @@ static size_t ZSTD_resetCCtx_usingCDict(ZSTD_CCtx* cctx,
         32 KB, /* ZSTD_btopt */
         8 KB /* ZSTD_btultra */
     };
-    const int attachDict = ( pledgedSrcSize <= attachDictSizeCutoffs[cdict->cParams.strategy]
+    const ZSTD_compressionParameters *cdict_cParams = &cdict->matchState.cParams;
+    const int attachDict = ( pledgedSrcSize <= attachDictSizeCutoffs[cdict_cParams->strategy]
                           || pledgedSrcSize == ZSTD_CONTENTSIZE_UNKNOWN
                           || params.attachDictPref == ZSTD_dictForceAttach )
                         && params.attachDictPref != ZSTD_dictForceCopy
                         && !params.forceWindow /* dictMatchState isn't correctly
                                                 * handled in _enforceMaxDist */
                         && ZSTD_equivalentCParams(cctx->appliedParams.cParams,
-                                                  cdict->cParams);
+                                                  *cdict_cParams);
 
     DEBUGLOG(4, "ZSTD_resetCCtx_usingCDict (pledgedSrcSize=%u)", (U32)pledgedSrcSize);
 
@@ -1322,14 +1322,14 @@ static size_t ZSTD_resetCCtx_usingCDict(ZSTD_CCtx* cctx,
     {   unsigned const windowLog = params.cParams.windowLog;
         assert(windowLog != 0);
         /* Copy only compression parameters related to tables. */
-        params.cParams = cdict->cParams;
+        params.cParams = *cdict_cParams;
         params.cParams.windowLog = windowLog;
         ZSTD_resetCCtx_internal(cctx, params, pledgedSrcSize,
                                 attachDict ? ZSTDcrp_continue : ZSTDcrp_noMemset,
                                 zbuff);
-        assert(cctx->appliedParams.cParams.strategy == cdict->cParams.strategy);
-        assert(cctx->appliedParams.cParams.hashLog == cdict->cParams.hashLog);
-        assert(cctx->appliedParams.cParams.chainLog == cdict->cParams.chainLog);
+        assert(cctx->appliedParams.cParams.strategy == cdict_cParams->strategy);
+        assert(cctx->appliedParams.cParams.hashLog == cdict_cParams->hashLog);
+        assert(cctx->appliedParams.cParams.chainLog == cdict_cParams->chainLog);
     }
 
     if (attachDict) {
@@ -1355,8 +1355,8 @@ static size_t ZSTD_resetCCtx_usingCDict(ZSTD_CCtx* cctx,
     } else {
         DEBUGLOG(4, "copying dictionary into context");
         /* copy tables */
-        {   size_t const chainSize = (cdict->cParams.strategy == ZSTD_fast) ? 0 : ((size_t)1 << cdict->cParams.chainLog);
-            size_t const hSize =  (size_t)1 << cdict->cParams.hashLog;
+        {   size_t const chainSize = (cdict_cParams->strategy == ZSTD_fast) ? 0 : ((size_t)1 << cdict_cParams->chainLog);
+            size_t const hSize =  (size_t)1 << cdict_cParams->hashLog;
             size_t const tableSpace = (chainSize + hSize) * sizeof(U32);
             assert((U32*)cctx->blockState.matchState.chainTable == (U32*)cctx->blockState.matchState.hashTable + hSize);  /* chainTable must follow hashTable */
             assert((U32*)cctx->blockState.matchState.hashTable3 == (U32*)cctx->blockState.matchState.chainTable + chainSize);
@@ -3143,7 +3143,7 @@ static size_t ZSTD_initCDict_internal(
 {
     DEBUGLOG(3, "ZSTD_initCDict_internal (dictContentType:%u)", (U32)dictContentType);
     assert(!ZSTD_checkCParams(cParams));
-    cdict->cParams = cParams;
+    cdict->matchState.cParams = cParams;
     if ((dictLoadMethod == ZSTD_dlm_byRef) || (!dictBuffer) || (!dictSize)) {
         cdict->dictBuffer = NULL;
         cdict->dictContent = dictBuffer;
@@ -3297,7 +3297,7 @@ const ZSTD_CDict* ZSTD_initStaticCDict(
 ZSTD_compressionParameters ZSTD_getCParamsFromCDict(const ZSTD_CDict* cdict)
 {
     assert(cdict != NULL);
-    return cdict->cParams;
+    return cdict->matchState.cParams;
 }
 
 /* ZSTD_compressBegin_usingCDict_advanced() :

From 03103269de51fdc5853d073ce7994fce6491ec73 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" 
Date: Thu, 23 Aug 2018 11:38:35 -0700
Subject: [PATCH 327/372] Assert `ctx` and `ms` cparams Equivalency

---
 lib/compress/zstd_compress.c | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index ae3626b6d..9e16845f2 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -2356,6 +2356,9 @@ static size_t ZSTD_compressBlock_internal(ZSTD_CCtx* zc,
                 dstCapacity, ms->window.dictLimit, ms->nextToUpdate);
     assert(srcSize <= ZSTD_BLOCKSIZE_MAX);
 
+    /* Assert that we have correctly flushed the ctx params into the ms's copy */
+    assert(ZSTD_equivalentCParams(zc->appliedParams.cParams, ms->cParams));
+
     if (srcSize < MIN_CBLOCK_SIZE+ZSTD_blockHeaderSize+1) {
         ZSTD_ldm_skipSequences(&zc->externSeqStore, srcSize, zc->appliedParams.cParams.searchLength);
         return 0;   /* don't even attempt compression below a certain srcSize */

From 6cb24546465f18f1a7b1c179ade84c9b34184b65 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" 
Date: Thu, 23 Aug 2018 11:53:34 -0700
Subject: [PATCH 328/372] Remove CParams from Block Compressor Functions' Args

---
 lib/compress/zstd_compress.c          |  2 +-
 lib/compress/zstd_compress_internal.h |  2 +-
 lib/compress/zstd_double_fast.c       | 42 +++++++++++-----------
 lib/compress/zstd_double_fast.h       |  6 ++--
 lib/compress/zstd_fast.c              |  9 +++--
 lib/compress/zstd_fast.h              |  6 ++--
 lib/compress/zstd_lazy.c              | 52 +++++++++++++--------------
 lib/compress/zstd_lazy.h              | 24 ++++++-------
 lib/compress/zstd_ldm.c               |  4 +--
 lib/compress/zstd_opt.c               | 28 +++++++--------
 lib/compress/zstd_opt.h               | 12 +++----
 11 files changed, 96 insertions(+), 91 deletions(-)

diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index 9e16845f2..b9785195f 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -2416,7 +2416,7 @@ static size_t ZSTD_compressBlock_internal(ZSTD_CCtx* zc,
             assert(ldmSeqStore.pos == ldmSeqStore.size);
         } else {   /* not long range mode */
             ZSTD_blockCompressor const blockCompressor = ZSTD_selectBlockCompressor(zc->appliedParams.cParams.strategy, dictMode);
-            lastLLSize = blockCompressor(ms, &zc->seqStore, zc->blockState.nextCBlock->rep, &zc->appliedParams.cParams, src, srcSize);
+            lastLLSize = blockCompressor(ms, &zc->seqStore, zc->blockState.nextCBlock->rep, src, srcSize);
         }
         {   const BYTE* const lastLiterals = (const BYTE*)src + srcSize - lastLLSize;
             ZSTD_storeLastLiterals(&zc->seqStore, lastLiterals, lastLLSize);
diff --git a/lib/compress/zstd_compress_internal.h b/lib/compress/zstd_compress_internal.h
index 1c45d1b07..43f7c1486 100644
--- a/lib/compress/zstd_compress_internal.h
+++ b/lib/compress/zstd_compress_internal.h
@@ -265,7 +265,7 @@ typedef enum { ZSTD_noDict = 0, ZSTD_extDict = 1, ZSTD_dictMatchState = 2 } ZSTD
 
 typedef size_t (*ZSTD_blockCompressor) (
         ZSTD_matchState_t* bs, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
-        ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize);
+        void const* src, size_t srcSize);
 ZSTD_blockCompressor ZSTD_selectBlockCompressor(ZSTD_strategy strat, ZSTD_dictMode_e dictMode);
 
 
diff --git a/lib/compress/zstd_double_fast.c b/lib/compress/zstd_double_fast.c
index 9231c8157..9ce3560fc 100644
--- a/lib/compress/zstd_double_fast.c
+++ b/lib/compress/zstd_double_fast.c
@@ -51,9 +51,10 @@ void ZSTD_fillDoubleHashTable(ZSTD_matchState_t* ms,
 FORCE_INLINE_TEMPLATE
 size_t ZSTD_compressBlock_doubleFast_generic(
         ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
-        ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize,
+        void const* src, size_t srcSize,
         U32 const mls /* template */, ZSTD_dictMode_e const dictMode)
 {
+    ZSTD_compressionParameters const* cParams = &ms->cParams;
     U32* const hashLong = ms->hashTable;
     const U32 hBitsL = cParams->hashLog;
     U32* const hashSmall = ms->chainTable;
@@ -296,49 +297,50 @@ _match_stored:
 
 size_t ZSTD_compressBlock_doubleFast(
         ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
-        ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize)
+        void const* src, size_t srcSize)
 {
-    const U32 mls = cParams->searchLength;
+    const U32 mls = ms->cParams.searchLength;
     switch(mls)
     {
     default: /* includes case 3 */
     case 4 :
-        return ZSTD_compressBlock_doubleFast_generic(ms, seqStore, rep, cParams, src, srcSize, 4, ZSTD_noDict);
+        return ZSTD_compressBlock_doubleFast_generic(ms, seqStore, rep, src, srcSize, 4, ZSTD_noDict);
     case 5 :
-        return ZSTD_compressBlock_doubleFast_generic(ms, seqStore, rep, cParams, src, srcSize, 5, ZSTD_noDict);
+        return ZSTD_compressBlock_doubleFast_generic(ms, seqStore, rep, src, srcSize, 5, ZSTD_noDict);
     case 6 :
-        return ZSTD_compressBlock_doubleFast_generic(ms, seqStore, rep, cParams, src, srcSize, 6, ZSTD_noDict);
+        return ZSTD_compressBlock_doubleFast_generic(ms, seqStore, rep, src, srcSize, 6, ZSTD_noDict);
     case 7 :
-        return ZSTD_compressBlock_doubleFast_generic(ms, seqStore, rep, cParams, src, srcSize, 7, ZSTD_noDict);
+        return ZSTD_compressBlock_doubleFast_generic(ms, seqStore, rep, src, srcSize, 7, ZSTD_noDict);
     }
 }
 
 
 size_t ZSTD_compressBlock_doubleFast_dictMatchState(
         ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
-        ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize)
+        void const* src, size_t srcSize)
 {
-    const U32 mls = cParams->searchLength;
+    const U32 mls = ms->cParams.searchLength;
     switch(mls)
     {
     default: /* includes case 3 */
     case 4 :
-        return ZSTD_compressBlock_doubleFast_generic(ms, seqStore, rep, cParams, src, srcSize, 4, ZSTD_dictMatchState);
+        return ZSTD_compressBlock_doubleFast_generic(ms, seqStore, rep, src, srcSize, 4, ZSTD_dictMatchState);
     case 5 :
-        return ZSTD_compressBlock_doubleFast_generic(ms, seqStore, rep, cParams, src, srcSize, 5, ZSTD_dictMatchState);
+        return ZSTD_compressBlock_doubleFast_generic(ms, seqStore, rep, src, srcSize, 5, ZSTD_dictMatchState);
     case 6 :
-        return ZSTD_compressBlock_doubleFast_generic(ms, seqStore, rep, cParams, src, srcSize, 6, ZSTD_dictMatchState);
+        return ZSTD_compressBlock_doubleFast_generic(ms, seqStore, rep, src, srcSize, 6, ZSTD_dictMatchState);
     case 7 :
-        return ZSTD_compressBlock_doubleFast_generic(ms, seqStore, rep, cParams, src, srcSize, 7, ZSTD_dictMatchState);
+        return ZSTD_compressBlock_doubleFast_generic(ms, seqStore, rep, src, srcSize, 7, ZSTD_dictMatchState);
     }
 }
 
 
 static size_t ZSTD_compressBlock_doubleFast_extDict_generic(
         ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
-        ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize,
+        void const* src, size_t srcSize,
         U32 const mls /* template */)
 {
+    ZSTD_compressionParameters const* cParams = &ms->cParams;
     U32* const hashLong = ms->hashTable;
     U32  const hBitsL = cParams->hashLog;
     U32* const hashSmall = ms->chainTable;
@@ -469,19 +471,19 @@ static size_t ZSTD_compressBlock_doubleFast_extDict_generic(
 
 size_t ZSTD_compressBlock_doubleFast_extDict(
         ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
-        ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize)
+        void const* src, size_t srcSize)
 {
-    U32 const mls = cParams->searchLength;
+    U32 const mls = ms->cParams.searchLength;
     switch(mls)
     {
     default: /* includes case 3 */
     case 4 :
-        return ZSTD_compressBlock_doubleFast_extDict_generic(ms, seqStore, rep, cParams, src, srcSize, 4);
+        return ZSTD_compressBlock_doubleFast_extDict_generic(ms, seqStore, rep, src, srcSize, 4);
     case 5 :
-        return ZSTD_compressBlock_doubleFast_extDict_generic(ms, seqStore, rep, cParams, src, srcSize, 5);
+        return ZSTD_compressBlock_doubleFast_extDict_generic(ms, seqStore, rep, src, srcSize, 5);
     case 6 :
-        return ZSTD_compressBlock_doubleFast_extDict_generic(ms, seqStore, rep, cParams, src, srcSize, 6);
+        return ZSTD_compressBlock_doubleFast_extDict_generic(ms, seqStore, rep, src, srcSize, 6);
     case 7 :
-        return ZSTD_compressBlock_doubleFast_extDict_generic(ms, seqStore, rep, cParams, src, srcSize, 7);
+        return ZSTD_compressBlock_doubleFast_extDict_generic(ms, seqStore, rep, src, srcSize, 7);
     }
 }
diff --git a/lib/compress/zstd_double_fast.h b/lib/compress/zstd_double_fast.h
index c475021d2..d12b15da0 100644
--- a/lib/compress/zstd_double_fast.h
+++ b/lib/compress/zstd_double_fast.h
@@ -23,13 +23,13 @@ void ZSTD_fillDoubleHashTable(ZSTD_matchState_t* ms,
                               void const* end, ZSTD_dictTableLoadMethod_e dtlm);
 size_t ZSTD_compressBlock_doubleFast(
         ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
-        ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize);
+        void const* src, size_t srcSize);
 size_t ZSTD_compressBlock_doubleFast_dictMatchState(
         ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
-        ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize);
+        void const* src, size_t srcSize);
 size_t ZSTD_compressBlock_doubleFast_extDict(
         ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
-        ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize);
+        void const* src, size_t srcSize);
 
 
 #if defined (__cplusplus)
diff --git a/lib/compress/zstd_fast.c b/lib/compress/zstd_fast.c
index 8d2c8452a..91eb85827 100644
--- a/lib/compress/zstd_fast.c
+++ b/lib/compress/zstd_fast.c
@@ -224,8 +224,9 @@ size_t ZSTD_compressBlock_fast_generic(
 
 size_t ZSTD_compressBlock_fast(
         ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
-        ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize)
+        void const* src, size_t srcSize)
 {
+    ZSTD_compressionParameters const* cParams = &ms->cParams;
     U32 const hlog = cParams->hashLog;
     U32 const mls = cParams->searchLength;
     U32 const stepSize = cParams->targetLength;
@@ -246,8 +247,9 @@ size_t ZSTD_compressBlock_fast(
 
 size_t ZSTD_compressBlock_fast_dictMatchState(
         ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
-        ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize)
+        void const* src, size_t srcSize)
 {
+    ZSTD_compressionParameters const* cParams = &ms->cParams;
     U32 const hlog = cParams->hashLog;
     U32 const mls = cParams->searchLength;
     U32 const stepSize = cParams->targetLength;
@@ -365,8 +367,9 @@ static size_t ZSTD_compressBlock_fast_extDict_generic(
 
 size_t ZSTD_compressBlock_fast_extDict(
         ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
-        ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize)
+        void const* src, size_t srcSize)
 {
+    ZSTD_compressionParameters const* cParams = &ms->cParams;
     U32 const hlog = cParams->hashLog;
     U32 const mls = cParams->searchLength;
     U32 const stepSize = cParams->targetLength;
diff --git a/lib/compress/zstd_fast.h b/lib/compress/zstd_fast.h
index 7e7435f8c..dbbf86a55 100644
--- a/lib/compress/zstd_fast.h
+++ b/lib/compress/zstd_fast.h
@@ -23,13 +23,13 @@ void ZSTD_fillHashTable(ZSTD_matchState_t* ms,
                         void const* end, ZSTD_dictTableLoadMethod_e dtlm);
 size_t ZSTD_compressBlock_fast(
         ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
-        ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize);
+        void const* src, size_t srcSize);
 size_t ZSTD_compressBlock_fast_dictMatchState(
         ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
-        ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize);
+        void const* src, size_t srcSize);
 size_t ZSTD_compressBlock_fast_extDict(
         ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
-        ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize);
+        void const* src, size_t srcSize);
 
 #if defined (__cplusplus)
 }
diff --git a/lib/compress/zstd_lazy.c b/lib/compress/zstd_lazy.c
index f17fced15..078a0c138 100644
--- a/lib/compress/zstd_lazy.c
+++ b/lib/compress/zstd_lazy.c
@@ -596,7 +596,6 @@ FORCE_INLINE_TEMPLATE
 size_t ZSTD_compressBlock_lazy_generic(
                         ZSTD_matchState_t* ms, seqStore_t* seqStore,
                         U32 rep[ZSTD_REP_NUM],
-                        ZSTD_compressionParameters const* cParams,
                         const void* src, size_t srcSize,
                         const U32 searchMethod, const U32 depth,
                         ZSTD_dictMode_e const dictMode)
@@ -609,6 +608,7 @@ size_t ZSTD_compressBlock_lazy_generic(
     const BYTE* const base = ms->window.base;
     const U32 prefixLowestIndex = ms->window.dictLimit;
     const BYTE* const prefixLowest = base + prefixLowestIndex;
+    const ZSTD_compressionParameters* cParams = &ms->cParams;
 
     typedef size_t (*searchMax_f)(
                         ZSTD_matchState_t* ms, ZSTD_compressionParameters const* cParams,
@@ -833,58 +833,58 @@ _storeSequence:
 
 size_t ZSTD_compressBlock_btlazy2(
         ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
-        ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize)
+        void const* src, size_t srcSize)
 {
-    return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, cParams, src, srcSize, 1, 2, ZSTD_noDict);
+    return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, 1, 2, ZSTD_noDict);
 }
 
 size_t ZSTD_compressBlock_lazy2(
         ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
-        ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize)
+        void const* src, size_t srcSize)
 {
-    return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, cParams, src, srcSize, 0, 2, ZSTD_noDict);
+    return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, 0, 2, ZSTD_noDict);
 }
 
 size_t ZSTD_compressBlock_lazy(
         ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
-        ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize)
+        void const* src, size_t srcSize)
 {
-    return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, cParams, src, srcSize, 0, 1, ZSTD_noDict);
+    return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, 0, 1, ZSTD_noDict);
 }
 
 size_t ZSTD_compressBlock_greedy(
         ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
-        ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize)
+        void const* src, size_t srcSize)
 {
-    return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, cParams, src, srcSize, 0, 0, ZSTD_noDict);
+    return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, 0, 0, ZSTD_noDict);
 }
 
 size_t ZSTD_compressBlock_btlazy2_dictMatchState(
         ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
-        ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize)
+        void const* src, size_t srcSize)
 {
-    return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, cParams, src, srcSize, 1, 2, ZSTD_dictMatchState);
+    return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, 1, 2, ZSTD_dictMatchState);
 }
 
 size_t ZSTD_compressBlock_lazy2_dictMatchState(
         ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
-        ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize)
+        void const* src, size_t srcSize)
 {
-    return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, cParams, src, srcSize, 0, 2, ZSTD_dictMatchState);
+    return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, 0, 2, ZSTD_dictMatchState);
 }
 
 size_t ZSTD_compressBlock_lazy_dictMatchState(
         ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
-        ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize)
+        void const* src, size_t srcSize)
 {
-    return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, cParams, src, srcSize, 0, 1, ZSTD_dictMatchState);
+    return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, 0, 1, ZSTD_dictMatchState);
 }
 
 size_t ZSTD_compressBlock_greedy_dictMatchState(
         ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
-        ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize)
+        void const* src, size_t srcSize)
 {
-    return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, cParams, src, srcSize, 0, 0, ZSTD_dictMatchState);
+    return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, 0, 0, ZSTD_dictMatchState);
 }
 
 
@@ -892,7 +892,6 @@ FORCE_INLINE_TEMPLATE
 size_t ZSTD_compressBlock_lazy_extDict_generic(
                         ZSTD_matchState_t* ms, seqStore_t* seqStore,
                         U32 rep[ZSTD_REP_NUM],
-                        ZSTD_compressionParameters const* cParams,
                         const void* src, size_t srcSize,
                         const U32 searchMethod, const U32 depth)
 {
@@ -908,6 +907,7 @@ size_t ZSTD_compressBlock_lazy_extDict_generic(
     const BYTE* const dictBase = ms->window.dictBase;
     const BYTE* const dictEnd  = dictBase + dictLimit;
     const BYTE* const dictStart  = dictBase + lowestIndex;
+    const ZSTD_compressionParameters* const cParams = &ms->cParams;
 
     typedef size_t (*searchMax_f)(
                         ZSTD_matchState_t* ms, ZSTD_compressionParameters const* cParams,
@@ -1060,31 +1060,31 @@ _storeSequence:
 
 size_t ZSTD_compressBlock_greedy_extDict(
         ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
-        ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize)
+        void const* src, size_t srcSize)
 {
-    return ZSTD_compressBlock_lazy_extDict_generic(ms, seqStore, rep, cParams, src, srcSize, 0, 0);
+    return ZSTD_compressBlock_lazy_extDict_generic(ms, seqStore, rep, src, srcSize, 0, 0);
 }
 
 size_t ZSTD_compressBlock_lazy_extDict(
         ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
-        ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize)
+        void const* src, size_t srcSize)
 
 {
-    return ZSTD_compressBlock_lazy_extDict_generic(ms, seqStore, rep, cParams, src, srcSize, 0, 1);
+    return ZSTD_compressBlock_lazy_extDict_generic(ms, seqStore, rep, src, srcSize, 0, 1);
 }
 
 size_t ZSTD_compressBlock_lazy2_extDict(
         ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
-        ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize)
+        void const* src, size_t srcSize)
 
 {
-    return ZSTD_compressBlock_lazy_extDict_generic(ms, seqStore, rep, cParams, src, srcSize, 0, 2);
+    return ZSTD_compressBlock_lazy_extDict_generic(ms, seqStore, rep, src, srcSize, 0, 2);
 }
 
 size_t ZSTD_compressBlock_btlazy2_extDict(
         ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
-        ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize)
+        void const* src, size_t srcSize)
 
 {
-    return ZSTD_compressBlock_lazy_extDict_generic(ms, seqStore, rep, cParams, src, srcSize, 1, 2);
+    return ZSTD_compressBlock_lazy_extDict_generic(ms, seqStore, rep, src, srcSize, 1, 2);
 }
diff --git a/lib/compress/zstd_lazy.h b/lib/compress/zstd_lazy.h
index c299de6dc..ba3910e73 100644
--- a/lib/compress/zstd_lazy.h
+++ b/lib/compress/zstd_lazy.h
@@ -25,42 +25,42 @@ void ZSTD_preserveUnsortedMark (U32* const table, U32 const size, U32 const redu
 
 size_t ZSTD_compressBlock_btlazy2(
         ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
-        ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize);
+        void const* src, size_t srcSize);
 size_t ZSTD_compressBlock_lazy2(
         ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
-        ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize);
+        void const* src, size_t srcSize);
 size_t ZSTD_compressBlock_lazy(
         ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
-        ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize);
+        void const* src, size_t srcSize);
 size_t ZSTD_compressBlock_greedy(
         ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
-        ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize);
+        void const* src, size_t srcSize);
 
 size_t ZSTD_compressBlock_btlazy2_dictMatchState(
         ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
-        ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize);
+        void const* src, size_t srcSize);
 size_t ZSTD_compressBlock_lazy2_dictMatchState(
         ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
-        ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize);
+        void const* src, size_t srcSize);
 size_t ZSTD_compressBlock_lazy_dictMatchState(
         ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
-        ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize);
+        void const* src, size_t srcSize);
 size_t ZSTD_compressBlock_greedy_dictMatchState(
         ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
-        ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize);
+        void const* src, size_t srcSize);
 
 size_t ZSTD_compressBlock_greedy_extDict(
         ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
-        ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize);
+        void const* src, size_t srcSize);
 size_t ZSTD_compressBlock_lazy_extDict(
         ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
-        ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize);
+        void const* src, size_t srcSize);
 size_t ZSTD_compressBlock_lazy2_extDict(
         ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
-        ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize);
+        void const* src, size_t srcSize);
 size_t ZSTD_compressBlock_btlazy2_extDict(
         ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
-        ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize);
+        void const* src, size_t srcSize);
 
 #if defined (__cplusplus)
 }
diff --git a/lib/compress/zstd_ldm.c b/lib/compress/zstd_ldm.c
index 215f55cf4..88cbf85fc 100644
--- a/lib/compress/zstd_ldm.c
+++ b/lib/compress/zstd_ldm.c
@@ -625,7 +625,7 @@ size_t ZSTD_ldm_blockCompress(rawSeqStore_t* rawSeqStore,
         DEBUGLOG(5, "calling block compressor on segment of size %u", sequence.litLength);
         {
             size_t const newLitLength =
-                blockCompressor(ms, seqStore, rep, cParams, ip,
+                blockCompressor(ms, seqStore, rep, ip,
                                 sequence.litLength);
             ip += sequence.litLength;
             /* Update the repcodes */
@@ -643,6 +643,6 @@ size_t ZSTD_ldm_blockCompress(rawSeqStore_t* rawSeqStore,
     ZSTD_ldm_limitTableUpdate(ms, ip);
     ZSTD_ldm_fillFastTables(ms, cParams, ip);
     /* Compress the last literals */
-    return blockCompressor(ms, seqStore, rep, cParams,
+    return blockCompressor(ms, seqStore, rep,
                            ip, iend - ip);
 }
diff --git a/lib/compress/zstd_opt.c b/lib/compress/zstd_opt.c
index 694deec78..66a1c00ca 100644
--- a/lib/compress/zstd_opt.c
+++ b/lib/compress/zstd_opt.c
@@ -772,7 +772,6 @@ FORCE_INLINE_TEMPLATE size_t
 ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms,
                                seqStore_t* seqStore,
                                U32 rep[ZSTD_REP_NUM],
-                               const ZSTD_compressionParameters* cParams,
                                const void* src, size_t srcSize,
                                const int optLevel, const ZSTD_dictMode_e dictMode)
 {
@@ -784,6 +783,7 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms,
     const BYTE* const ilimit = iend - 8;
     const BYTE* const base = ms->window.base;
     const BYTE* const prefixStart = base + ms->window.dictLimit;
+    const ZSTD_compressionParameters* const cParams = &ms->cParams;
 
     U32 const sufficient_len = MIN(cParams->targetLength, ZSTD_OPT_NUM -1);
     U32 const minMatch = (cParams->searchLength == 3) ? 3 : 4;
@@ -1033,10 +1033,10 @@ _shortestPath:   /* cur, last_pos, best_mlen, best_off have to be set */
 
 size_t ZSTD_compressBlock_btopt(
         ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
-        const ZSTD_compressionParameters* cParams, const void* src, size_t srcSize)
+        const void* src, size_t srcSize)
 {
     DEBUGLOG(5, "ZSTD_compressBlock_btopt");
-    return ZSTD_compressBlock_opt_generic(ms, seqStore, rep, cParams, src, srcSize, 0 /*optLevel*/, ZSTD_noDict);
+    return ZSTD_compressBlock_opt_generic(ms, seqStore, rep, src, srcSize, 0 /*optLevel*/, ZSTD_noDict);
 }
 
 
@@ -1064,7 +1064,7 @@ MEM_STATIC void ZSTD_upscaleStats(optState_t* optPtr)
 
 size_t ZSTD_compressBlock_btultra(
         ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
-        const ZSTD_compressionParameters* cParams, const void* src, size_t srcSize)
+        const void* src, size_t srcSize)
 {
     DEBUGLOG(5, "ZSTD_compressBlock_btultra (srcSize=%zu)", srcSize);
 #if 0
@@ -1082,7 +1082,7 @@ size_t ZSTD_compressBlock_btultra(
         assert(ms->nextToUpdate >= ms->window.dictLimit
             && ms->nextToUpdate <= ms->window.dictLimit + 1);
         memcpy(tmpRep, rep, sizeof(tmpRep));
-        ZSTD_compressBlock_opt_generic(ms, seqStore, tmpRep, cParams, src, srcSize, 2 /*optLevel*/, ZSTD_noDict);   /* generate stats into ms->opt*/
+        ZSTD_compressBlock_opt_generic(ms, seqStore, tmpRep, src, srcSize, 2 /*optLevel*/, ZSTD_noDict);   /* generate stats into ms->opt*/
         ZSTD_resetSeqStore(seqStore);
         /* invalidate first scan from history */
         ms->window.base -= srcSize;
@@ -1094,33 +1094,33 @@ size_t ZSTD_compressBlock_btultra(
         ZSTD_upscaleStats(&ms->opt);
     }
 #endif
-    return ZSTD_compressBlock_opt_generic(ms, seqStore, rep, cParams, src, srcSize, 2 /*optLevel*/, ZSTD_noDict);
+    return ZSTD_compressBlock_opt_generic(ms, seqStore, rep, src, srcSize, 2 /*optLevel*/, ZSTD_noDict);
 }
 
 size_t ZSTD_compressBlock_btopt_dictMatchState(
         ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
-        const ZSTD_compressionParameters* cParams, const void* src, size_t srcSize)
+        const void* src, size_t srcSize)
 {
-    return ZSTD_compressBlock_opt_generic(ms, seqStore, rep, cParams, src, srcSize, 0 /*optLevel*/, ZSTD_dictMatchState);
+    return ZSTD_compressBlock_opt_generic(ms, seqStore, rep, src, srcSize, 0 /*optLevel*/, ZSTD_dictMatchState);
 }
 
 size_t ZSTD_compressBlock_btultra_dictMatchState(
         ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
-        const ZSTD_compressionParameters* cParams, const void* src, size_t srcSize)
+        const void* src, size_t srcSize)
 {
-    return ZSTD_compressBlock_opt_generic(ms, seqStore, rep, cParams, src, srcSize, 2 /*optLevel*/, ZSTD_dictMatchState);
+    return ZSTD_compressBlock_opt_generic(ms, seqStore, rep, src, srcSize, 2 /*optLevel*/, ZSTD_dictMatchState);
 }
 
 size_t ZSTD_compressBlock_btopt_extDict(
         ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
-        const ZSTD_compressionParameters* cParams, const void* src, size_t srcSize)
+        const void* src, size_t srcSize)
 {
-    return ZSTD_compressBlock_opt_generic(ms, seqStore, rep, cParams, src, srcSize, 0 /*optLevel*/, ZSTD_extDict);
+    return ZSTD_compressBlock_opt_generic(ms, seqStore, rep, src, srcSize, 0 /*optLevel*/, ZSTD_extDict);
 }
 
 size_t ZSTD_compressBlock_btultra_extDict(
         ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
-        const ZSTD_compressionParameters* cParams, const void* src, size_t srcSize)
+        const void* src, size_t srcSize)
 {
-    return ZSTD_compressBlock_opt_generic(ms, seqStore, rep, cParams, src, srcSize, 2 /*optLevel*/, ZSTD_extDict);
+    return ZSTD_compressBlock_opt_generic(ms, seqStore, rep, src, srcSize, 2 /*optLevel*/, ZSTD_extDict);
 }
diff --git a/lib/compress/zstd_opt.h b/lib/compress/zstd_opt.h
index 63dbe79a8..4d45a923f 100644
--- a/lib/compress/zstd_opt.h
+++ b/lib/compress/zstd_opt.h
@@ -23,24 +23,24 @@ void ZSTD_updateTree(
 
 size_t ZSTD_compressBlock_btopt(
         ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
-        ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize);
+        void const* src, size_t srcSize);
 size_t ZSTD_compressBlock_btultra(
         ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
-        ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize);
+        void const* src, size_t srcSize);
 
 size_t ZSTD_compressBlock_btopt_dictMatchState(
         ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
-        ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize);
+        void const* src, size_t srcSize);
 size_t ZSTD_compressBlock_btultra_dictMatchState(
         ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
-        ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize);
+        void const* src, size_t srcSize);
 
 size_t ZSTD_compressBlock_btopt_extDict(
         ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
-        ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize);
+        void const* src, size_t srcSize);
 size_t ZSTD_compressBlock_btultra_extDict(
         ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
-        ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize);
+        void const* src, size_t srcSize);
 
 #if defined (__cplusplus)
 }

From 3483f89101671d797a8776f4881555cbab80f932 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" 
Date: Thu, 23 Aug 2018 11:57:54 -0700
Subject: [PATCH 329/372] Also Assert Equivalency When Filling MatchState with
 Prefix

---
 lib/compress/zstd_compress.c | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index b9785195f..ecde50254 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -2700,6 +2700,9 @@ static size_t ZSTD_loadDictionaryContent(ZSTD_matchState_t* ms,
     ZSTD_window_update(&ms->window, src, srcSize);
     ms->loadedDictEnd = params->forceWindow ? 0 : (U32)(iend - ms->window.base);
 
+    /* Assert that we the ms params match the params we're being given */
+    assert(ZSTD_equivalentCParams(params->cParams, ms->cParams));
+
     if (srcSize <= HASH_READ_SIZE) return 0;
 
     switch(params->cParams.strategy)

From dcdf437fed7fa3c70b3a493a45bf9d0ffaef31b1 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" 
Date: Thu, 23 Aug 2018 12:08:03 -0700
Subject: [PATCH 330/372] Also Remove CParams from Table Filling Functions'
 Args

---
 lib/compress/zstd_compress.c    |  9 ++++-----
 lib/compress/zstd_double_fast.c |  2 +-
 lib/compress/zstd_double_fast.h |  1 -
 lib/compress/zstd_fast.c        |  2 +-
 lib/compress/zstd_fast.h        |  1 -
 lib/compress/zstd_lazy.c        | 12 +++++-------
 lib/compress/zstd_lazy.h        |  4 +---
 lib/compress/zstd_ldm.c         | 11 +++++------
 lib/compress/zstd_opt.c         | 12 +++++-------
 lib/compress/zstd_opt.h         |  5 ++---
 10 files changed, 24 insertions(+), 35 deletions(-)

diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index ecde50254..633400f7e 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -2695,7 +2695,6 @@ static size_t ZSTD_loadDictionaryContent(ZSTD_matchState_t* ms,
 {
     const BYTE* const ip = (const BYTE*) src;
     const BYTE* const iend = ip + srcSize;
-    ZSTD_compressionParameters const* cParams = ¶ms->cParams;
 
     ZSTD_window_update(&ms->window, src, srcSize);
     ms->loadedDictEnd = params->forceWindow ? 0 : (U32)(iend - ms->window.base);
@@ -2708,24 +2707,24 @@ static size_t ZSTD_loadDictionaryContent(ZSTD_matchState_t* ms,
     switch(params->cParams.strategy)
     {
     case ZSTD_fast:
-        ZSTD_fillHashTable(ms, cParams, iend, dtlm);
+        ZSTD_fillHashTable(ms, iend, dtlm);
         break;
     case ZSTD_dfast:
-        ZSTD_fillDoubleHashTable(ms, cParams, iend, dtlm);
+        ZSTD_fillDoubleHashTable(ms, iend, dtlm);
         break;
 
     case ZSTD_greedy:
     case ZSTD_lazy:
     case ZSTD_lazy2:
         if (srcSize >= HASH_READ_SIZE)
-            ZSTD_insertAndFindFirstIndex(ms, cParams, iend-HASH_READ_SIZE);
+            ZSTD_insertAndFindFirstIndex(ms, iend-HASH_READ_SIZE);
         break;
 
     case ZSTD_btlazy2:   /* we want the dictionary table fully sorted */
     case ZSTD_btopt:
     case ZSTD_btultra:
         if (srcSize >= HASH_READ_SIZE)
-            ZSTD_updateTree(ms, cParams, iend-HASH_READ_SIZE, iend);
+            ZSTD_updateTree(ms, iend-HASH_READ_SIZE, iend);
         break;
 
     default:
diff --git a/lib/compress/zstd_double_fast.c b/lib/compress/zstd_double_fast.c
index 9ce3560fc..e27104206 100644
--- a/lib/compress/zstd_double_fast.c
+++ b/lib/compress/zstd_double_fast.c
@@ -13,9 +13,9 @@
 
 
 void ZSTD_fillDoubleHashTable(ZSTD_matchState_t* ms,
-                              ZSTD_compressionParameters const* cParams,
                               void const* end, ZSTD_dictTableLoadMethod_e dtlm)
 {
+    const ZSTD_compressionParameters* const cParams = &ms->cParams;
     U32* const hashLarge = ms->hashTable;
     U32  const hBitsL = cParams->hashLog;
     U32  const mls = cParams->searchLength;
diff --git a/lib/compress/zstd_double_fast.h b/lib/compress/zstd_double_fast.h
index d12b15da0..4fa31acfc 100644
--- a/lib/compress/zstd_double_fast.h
+++ b/lib/compress/zstd_double_fast.h
@@ -19,7 +19,6 @@ extern "C" {
 #include "zstd_compress_internal.h"     /* ZSTD_CCtx, size_t */
 
 void ZSTD_fillDoubleHashTable(ZSTD_matchState_t* ms,
-                              ZSTD_compressionParameters const* cParams,
                               void const* end, ZSTD_dictTableLoadMethod_e dtlm);
 size_t ZSTD_compressBlock_doubleFast(
         ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
diff --git a/lib/compress/zstd_fast.c b/lib/compress/zstd_fast.c
index 91eb85827..28961db06 100644
--- a/lib/compress/zstd_fast.c
+++ b/lib/compress/zstd_fast.c
@@ -13,9 +13,9 @@
 
 
 void ZSTD_fillHashTable(ZSTD_matchState_t* ms,
-                        ZSTD_compressionParameters const* cParams,
                         void const* end, ZSTD_dictTableLoadMethod_e dtlm)
 {
+    const ZSTD_compressionParameters* const cParams = &ms->cParams;
     U32* const hashTable = ms->hashTable;
     U32  const hBits = cParams->hashLog;
     U32  const mls = cParams->searchLength;
diff --git a/lib/compress/zstd_fast.h b/lib/compress/zstd_fast.h
index dbbf86a55..b74a88c57 100644
--- a/lib/compress/zstd_fast.h
+++ b/lib/compress/zstd_fast.h
@@ -19,7 +19,6 @@ extern "C" {
 #include "zstd_compress_internal.h"
 
 void ZSTD_fillHashTable(ZSTD_matchState_t* ms,
-                        ZSTD_compressionParameters const* cParams,
                         void const* end, ZSTD_dictTableLoadMethod_e dtlm);
 size_t ZSTD_compressBlock_fast(
         ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
diff --git a/lib/compress/zstd_lazy.c b/lib/compress/zstd_lazy.c
index 078a0c138..dc0bfad74 100644
--- a/lib/compress/zstd_lazy.c
+++ b/lib/compress/zstd_lazy.c
@@ -427,9 +427,10 @@ static size_t ZSTD_BtFindBestMatch_extDict_selectMLS (
 /* Update chains up to ip (excluded)
    Assumption : always within prefix (i.e. not within extDict) */
 static U32 ZSTD_insertAndFindFirstIndex_internal(
-                        ZSTD_matchState_t* ms, ZSTD_compressionParameters const* cParams,
+                        ZSTD_matchState_t* ms,
                         const BYTE* ip, U32 const mls)
 {
+    const ZSTD_compressionParameters* const cParams = &ms->cParams;
     U32* const hashTable  = ms->hashTable;
     const U32 hashLog = cParams->hashLog;
     U32* const chainTable = ms->chainTable;
@@ -449,11 +450,8 @@ static U32 ZSTD_insertAndFindFirstIndex_internal(
     return hashTable[ZSTD_hashPtr(ip, hashLog, mls)];
 }
 
-U32 ZSTD_insertAndFindFirstIndex(
-                        ZSTD_matchState_t* ms, ZSTD_compressionParameters const* cParams,
-                        const BYTE* ip)
-{
-    return ZSTD_insertAndFindFirstIndex_internal(ms, cParams, ip, cParams->searchLength);
+U32 ZSTD_insertAndFindFirstIndex(ZSTD_matchState_t* ms, const BYTE* ip) {
+    return ZSTD_insertAndFindFirstIndex_internal(ms, ip, ms->cParams.searchLength);
 }
 
 
@@ -480,7 +478,7 @@ size_t ZSTD_HcFindBestMatch_generic (
     size_t ml=4-1;
 
     /* HC4 match finder */
-    U32 matchIndex = ZSTD_insertAndFindFirstIndex_internal(ms, cParams, ip, mls);
+    U32 matchIndex = ZSTD_insertAndFindFirstIndex_internal(ms, ip, mls);
 
     for ( ; (matchIndex>lowLimit) & (nbAttempts>0) ; nbAttempts--) {
         size_t currentMl=0;
diff --git a/lib/compress/zstd_lazy.h b/lib/compress/zstd_lazy.h
index ba3910e73..ef85a6df9 100644
--- a/lib/compress/zstd_lazy.h
+++ b/lib/compress/zstd_lazy.h
@@ -17,9 +17,7 @@ extern "C" {
 
 #include "zstd_compress_internal.h"
 
-U32 ZSTD_insertAndFindFirstIndex(
-        ZSTD_matchState_t* ms, ZSTD_compressionParameters const* cParams,
-        const BYTE* ip);
+U32 ZSTD_insertAndFindFirstIndex(ZSTD_matchState_t* ms, const BYTE* ip);
 
 void ZSTD_preserveUnsortedMark (U32* const table, U32 const size, U32 const reducerValue);  /*! used in ZSTD_reduceIndex(). pre-emptively increase value of ZSTD_DUBT_UNSORTED_MARK */
 
diff --git a/lib/compress/zstd_ldm.c b/lib/compress/zstd_ldm.c
index 88cbf85fc..99cbff980 100644
--- a/lib/compress/zstd_ldm.c
+++ b/lib/compress/zstd_ldm.c
@@ -218,19 +218,18 @@ static size_t ZSTD_ldm_countBackwardsMatch(
  *  The tables for the other strategies are filled within their
  *  block compressors. */
 static size_t ZSTD_ldm_fillFastTables(ZSTD_matchState_t* ms,
-                                      ZSTD_compressionParameters const* cParams,
                                       void const* end)
 {
     const BYTE* const iend = (const BYTE*)end;
 
-    switch(cParams->strategy)
+    switch(ms->cParams.strategy)
     {
     case ZSTD_fast:
-        ZSTD_fillHashTable(ms, cParams, iend, ZSTD_dtlm_fast);
+        ZSTD_fillHashTable(ms, iend, ZSTD_dtlm_fast);
         break;
 
     case ZSTD_dfast:
-        ZSTD_fillDoubleHashTable(ms, cParams, iend, ZSTD_dtlm_fast);
+        ZSTD_fillDoubleHashTable(ms, iend, ZSTD_dtlm_fast);
         break;
 
     case ZSTD_greedy:
@@ -620,7 +619,7 @@ size_t ZSTD_ldm_blockCompress(rawSeqStore_t* rawSeqStore,
 
         /* Fill tables for block compressor */
         ZSTD_ldm_limitTableUpdate(ms, ip);
-        ZSTD_ldm_fillFastTables(ms, cParams, ip);
+        ZSTD_ldm_fillFastTables(ms, ip);
         /* Run the block compressor */
         DEBUGLOG(5, "calling block compressor on segment of size %u", sequence.litLength);
         {
@@ -641,7 +640,7 @@ size_t ZSTD_ldm_blockCompress(rawSeqStore_t* rawSeqStore,
     }
     /* Fill the tables for the block compressor */
     ZSTD_ldm_limitTableUpdate(ms, ip);
-    ZSTD_ldm_fillFastTables(ms, cParams, ip);
+    ZSTD_ldm_fillFastTables(ms, ip);
     /* Compress the last literals */
     return blockCompressor(ms, seqStore, rep,
                            ip, iend - ip);
diff --git a/lib/compress/zstd_opt.c b/lib/compress/zstd_opt.c
index 66a1c00ca..b979751c3 100644
--- a/lib/compress/zstd_opt.c
+++ b/lib/compress/zstd_opt.c
@@ -471,10 +471,11 @@ static U32 ZSTD_insertBt1(
 
 FORCE_INLINE_TEMPLATE
 void ZSTD_updateTree_internal(
-                ZSTD_matchState_t* ms, ZSTD_compressionParameters const* cParams,
+                ZSTD_matchState_t* ms,
                 const BYTE* const ip, const BYTE* const iend,
                 const U32 mls, const ZSTD_dictMode_e dictMode)
 {
+    const ZSTD_compressionParameters* const cParams = &ms->cParams;
     const BYTE* const base = ms->window.base;
     U32 const target = (U32)(ip - base);
     U32 idx = ms->nextToUpdate;
@@ -486,11 +487,8 @@ void ZSTD_updateTree_internal(
     ms->nextToUpdate = target;
 }
 
-void ZSTD_updateTree(
-                ZSTD_matchState_t* ms, ZSTD_compressionParameters const* cParams,
-                const BYTE* ip, const BYTE* iend)
-{
-    ZSTD_updateTree_internal(ms, cParams, ip, iend, cParams->searchLength, ZSTD_noDict);
+void ZSTD_updateTree(ZSTD_matchState_t* ms, const BYTE* ip, const BYTE* iend) {
+    ZSTD_updateTree_internal(ms, ip, iend, ms->cParams.searchLength, ZSTD_noDict);
 }
 
 FORCE_INLINE_TEMPLATE
@@ -721,7 +719,7 @@ FORCE_INLINE_TEMPLATE U32 ZSTD_BtGetAllMatches (
     U32 const matchLengthSearch = cParams->searchLength;
     DEBUGLOG(8, "ZSTD_BtGetAllMatches");
     if (ip < ms->window.base + ms->nextToUpdate) return 0;   /* skipped area */
-    ZSTD_updateTree_internal(ms, cParams, ip, iHighLimit, matchLengthSearch, dictMode);
+    ZSTD_updateTree_internal(ms, ip, iHighLimit, matchLengthSearch, dictMode);
     switch(matchLengthSearch)
     {
     case 3 : return ZSTD_insertBtAndGetAllMatches(ms, cParams, ip, iHighLimit, dictMode, rep, ll0, matches, lengthToBeat, 3);
diff --git a/lib/compress/zstd_opt.h b/lib/compress/zstd_opt.h
index 4d45a923f..eeadb604c 100644
--- a/lib/compress/zstd_opt.h
+++ b/lib/compress/zstd_opt.h
@@ -17,9 +17,8 @@ extern "C" {
 
 #include "zstd_compress_internal.h"
 
-void ZSTD_updateTree(
-        ZSTD_matchState_t* ms, ZSTD_compressionParameters const* cParams,
-        const BYTE* ip, const BYTE* iend);  /* used in ZSTD_loadDictionaryContent() */
+/* used in ZSTD_loadDictionaryContent() */
+void ZSTD_updateTree(ZSTD_matchState_t* ms, const BYTE* ip, const BYTE* iend);
 
 size_t ZSTD_compressBlock_btopt(
         ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],

From 97149f22c3f80903da34221e863b6ac8195ca5bb Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" 
Date: Thu, 23 Aug 2018 12:11:20 -0700
Subject: [PATCH 331/372] Stop Separately Passing CParams in ZSTD_opt Internal
 Functions

---
 lib/compress/zstd_opt.c | 24 +++++++++++++-----------
 1 file changed, 13 insertions(+), 11 deletions(-)

diff --git a/lib/compress/zstd_opt.c b/lib/compress/zstd_opt.c
index b979751c3..25d3487a7 100644
--- a/lib/compress/zstd_opt.c
+++ b/lib/compress/zstd_opt.c
@@ -360,10 +360,11 @@ static U32 ZSTD_insertAndFindFirstIndexHash3 (ZSTD_matchState_t* ms, const BYTE*
  *  ip : assumed <= iend-8 .
  * @return : nb of positions added */
 static U32 ZSTD_insertBt1(
-                ZSTD_matchState_t* ms, ZSTD_compressionParameters const* cParams,
+                ZSTD_matchState_t* ms,
                 const BYTE* const ip, const BYTE* const iend,
                 U32 const mls, const int extDict)
 {
+    const ZSTD_compressionParameters* const cParams = &ms->cParams;
     U32*   const hashTable = ms->hashTable;
     U32    const hashLog = cParams->hashLog;
     size_t const h  = ZSTD_hashPtr(ip, hashLog, mls);
@@ -475,7 +476,6 @@ void ZSTD_updateTree_internal(
                 const BYTE* const ip, const BYTE* const iend,
                 const U32 mls, const ZSTD_dictMode_e dictMode)
 {
-    const ZSTD_compressionParameters* const cParams = &ms->cParams;
     const BYTE* const base = ms->window.base;
     U32 const target = (U32)(ip - base);
     U32 idx = ms->nextToUpdate;
@@ -483,7 +483,7 @@ void ZSTD_updateTree_internal(
                 idx, target, dictMode);
 
     while(idx < target)
-        idx += ZSTD_insertBt1(ms, cParams, base+idx, iend, mls, dictMode == ZSTD_extDict);
+        idx += ZSTD_insertBt1(ms, base+idx, iend, mls, dictMode == ZSTD_extDict);
     ms->nextToUpdate = target;
 }
 
@@ -493,11 +493,12 @@ void ZSTD_updateTree(ZSTD_matchState_t* ms, const BYTE* ip, const BYTE* iend) {
 
 FORCE_INLINE_TEMPLATE
 U32 ZSTD_insertBtAndGetAllMatches (
-                    ZSTD_matchState_t* ms, ZSTD_compressionParameters const* cParams,
+                    ZSTD_matchState_t* ms,
                     const BYTE* const ip, const BYTE* const iLimit, const ZSTD_dictMode_e dictMode,
                     U32 rep[ZSTD_REP_NUM], U32 const ll0,
                     ZSTD_match_t* matches, const U32 lengthToBeat, U32 const mls /* template */)
 {
+    const ZSTD_compressionParameters* const cParams = &ms->cParams;
     U32 const sufficient_len = MIN(cParams->targetLength, ZSTD_OPT_NUM -1);
     const BYTE* const base = ms->window.base;
     U32 const current = (U32)(ip-base);
@@ -711,23 +712,24 @@ U32 ZSTD_insertBtAndGetAllMatches (
 
 
 FORCE_INLINE_TEMPLATE U32 ZSTD_BtGetAllMatches (
-                        ZSTD_matchState_t* ms, ZSTD_compressionParameters const* cParams,
+                        ZSTD_matchState_t* ms,
                         const BYTE* ip, const BYTE* const iHighLimit, const ZSTD_dictMode_e dictMode,
                         U32 rep[ZSTD_REP_NUM], U32 const ll0,
                         ZSTD_match_t* matches, U32 const lengthToBeat)
 {
+    const ZSTD_compressionParameters* const cParams = &ms->cParams;
     U32 const matchLengthSearch = cParams->searchLength;
     DEBUGLOG(8, "ZSTD_BtGetAllMatches");
     if (ip < ms->window.base + ms->nextToUpdate) return 0;   /* skipped area */
     ZSTD_updateTree_internal(ms, ip, iHighLimit, matchLengthSearch, dictMode);
     switch(matchLengthSearch)
     {
-    case 3 : return ZSTD_insertBtAndGetAllMatches(ms, cParams, ip, iHighLimit, dictMode, rep, ll0, matches, lengthToBeat, 3);
+    case 3 : return ZSTD_insertBtAndGetAllMatches(ms, ip, iHighLimit, dictMode, rep, ll0, matches, lengthToBeat, 3);
     default :
-    case 4 : return ZSTD_insertBtAndGetAllMatches(ms, cParams, ip, iHighLimit, dictMode, rep, ll0, matches, lengthToBeat, 4);
-    case 5 : return ZSTD_insertBtAndGetAllMatches(ms, cParams, ip, iHighLimit, dictMode, rep, ll0, matches, lengthToBeat, 5);
+    case 4 : return ZSTD_insertBtAndGetAllMatches(ms, ip, iHighLimit, dictMode, rep, ll0, matches, lengthToBeat, 4);
+    case 5 : return ZSTD_insertBtAndGetAllMatches(ms, ip, iHighLimit, dictMode, rep, ll0, matches, lengthToBeat, 5);
     case 7 :
-    case 6 : return ZSTD_insertBtAndGetAllMatches(ms, cParams, ip, iHighLimit, dictMode, rep, ll0, matches, lengthToBeat, 6);
+    case 6 : return ZSTD_insertBtAndGetAllMatches(ms, ip, iHighLimit, dictMode, rep, ll0, matches, lengthToBeat, 6);
     }
 }
 
@@ -804,7 +806,7 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms,
         /* find first match */
         {   U32 const litlen = (U32)(ip - anchor);
             U32 const ll0 = !litlen;
-            U32 const nbMatches = ZSTD_BtGetAllMatches(ms, cParams, ip, iend, dictMode, rep, ll0, matches, minMatch);
+            U32 const nbMatches = ZSTD_BtGetAllMatches(ms, ip, iend, dictMode, rep, ll0, matches, minMatch);
             if (!nbMatches) { ip++; continue; }
 
             /* initialize opt[0] */
@@ -901,7 +903,7 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms,
                 U32 const litlen = (opt[cur].mlen == 0) ? opt[cur].litlen : 0;
                 U32 const previousPrice = opt[cur].price;
                 U32 const basePrice = previousPrice + ZSTD_litLengthPrice(0, optStatePtr, optLevel);
-                U32 const nbMatches = ZSTD_BtGetAllMatches(ms, cParams, inr, iend, dictMode, opt[cur].rep, ll0, matches, minMatch);
+                U32 const nbMatches = ZSTD_BtGetAllMatches(ms, inr, iend, dictMode, opt[cur].rep, ll0, matches, minMatch);
                 U32 matchNb;
                 if (!nbMatches) {
                     DEBUGLOG(7, "rPos:%u : no match found", cur);

From 14764de49fdf89cc9e912156bc5803364e8a22fd Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" 
Date: Thu, 23 Aug 2018 12:17:58 -0700
Subject: [PATCH 332/372] Stop Separately Passing CParams in ZSTD_lazy Internal
 Functions

---
 lib/compress/zstd_lazy.c | 105 ++++++++++++++++++++-------------------
 1 file changed, 53 insertions(+), 52 deletions(-)

diff --git a/lib/compress/zstd_lazy.c b/lib/compress/zstd_lazy.c
index dc0bfad74..725623d22 100644
--- a/lib/compress/zstd_lazy.c
+++ b/lib/compress/zstd_lazy.c
@@ -17,10 +17,11 @@
 ***************************************/
 
 static void ZSTD_updateDUBT(
-                ZSTD_matchState_t* ms, ZSTD_compressionParameters const* cParams,
+                ZSTD_matchState_t* ms,
                 const BYTE* ip, const BYTE* iend,
                 U32 mls)
 {
+    const ZSTD_compressionParameters* const cParams = &ms->cParams;
     U32* const hashTable = ms->hashTable;
     U32  const hashLog = cParams->hashLog;
 
@@ -60,10 +61,11 @@ static void ZSTD_updateDUBT(
  *  assumption : current >= btlow == (current - btmask)
  *  doesn't fail */
 static void ZSTD_insertDUBT1(
-                 ZSTD_matchState_t* ms, ZSTD_compressionParameters const* cParams,
+                 ZSTD_matchState_t* ms,
                  U32 current, const BYTE* inputEnd,
                  U32 nbCompares, U32 btLow, const ZSTD_dictMode_e dictMode)
 {
+    const ZSTD_compressionParameters* const cParams = &ms->cParams;
     U32*   const bt = ms->chainTable;
     U32    const btLog  = cParams->chainLog - 1;
     U32    const btMask = (1 << btLog) - 1;
@@ -141,13 +143,14 @@ static void ZSTD_insertDUBT1(
 
 
 static size_t ZSTD_DUBT_findBetterDictMatch (
-        ZSTD_matchState_t* ms, ZSTD_compressionParameters const* cParams,
+        ZSTD_matchState_t* ms,
         const BYTE* const ip, const BYTE* const iend,
         size_t* offsetPtr,
         size_t bestLength,
         U32 nbCompares,
         U32 const mls,
         const ZSTD_dictMode_e dictMode) {
+    const ZSTD_compressionParameters* const cParams = &ms->cParams;
     const ZSTD_matchState_t * const dms = ms->dictMatchState;
     const U32 * const dictHashTable = dms->hashTable;
     U32         const hashLog = cParams->hashLog;
@@ -219,12 +222,13 @@ static size_t ZSTD_DUBT_findBetterDictMatch (
 
 
 static size_t ZSTD_DUBT_findBestMatch (
-                            ZSTD_matchState_t* ms, ZSTD_compressionParameters const* cParams,
+                            ZSTD_matchState_t* ms,
                             const BYTE* const ip, const BYTE* const iend,
                             size_t* offsetPtr,
                             U32 const mls,
                             const ZSTD_dictMode_e dictMode)
 {
+    const ZSTD_compressionParameters* const cParams = &ms->cParams;
     U32*   const hashTable = ms->hashTable;
     U32    const hashLog = cParams->hashLog;
     size_t const h  = ZSTD_hashPtr(ip, hashLog, mls);
@@ -275,7 +279,7 @@ static size_t ZSTD_DUBT_findBestMatch (
     while (matchIndex) {  /* will end on matchIndex == 0 */
         U32* const nextCandidateIdxPtr = bt + 2*(matchIndex&btMask) + 1;
         U32 const nextCandidateIdx = *nextCandidateIdxPtr;
-        ZSTD_insertDUBT1(ms, cParams, matchIndex, iend,
+        ZSTD_insertDUBT1(ms, matchIndex, iend,
                          nbCandidates, unsortLimit, dictMode);
         matchIndex = nextCandidateIdx;
         nbCandidates++;
@@ -340,7 +344,7 @@ static size_t ZSTD_DUBT_findBestMatch (
         *smallerPtr = *largerPtr = 0;
 
         if (dictMode == ZSTD_dictMatchState && nbCompares) {
-            bestLength = ZSTD_DUBT_findBetterDictMatch(ms, cParams, ip, iend, offsetPtr, bestLength, nbCompares, mls, dictMode);
+            bestLength = ZSTD_DUBT_findBetterDictMatch(ms, ip, iend, offsetPtr, bestLength, nbCompares, mls, dictMode);
         }
 
         assert(matchEndIdx > current+8); /* ensure nextToUpdate is increased */
@@ -357,7 +361,7 @@ static size_t ZSTD_DUBT_findBestMatch (
 
 /** ZSTD_BtFindBestMatch() : Tree updater, providing best match */
 FORCE_INLINE_TEMPLATE size_t ZSTD_BtFindBestMatch (
-                        ZSTD_matchState_t* ms, ZSTD_compressionParameters const* cParams,
+                        ZSTD_matchState_t* ms,
                         const BYTE* const ip, const BYTE* const iLimit,
                         size_t* offsetPtr,
                         const U32 mls /* template */,
@@ -365,55 +369,55 @@ FORCE_INLINE_TEMPLATE size_t ZSTD_BtFindBestMatch (
 {
     DEBUGLOG(7, "ZSTD_BtFindBestMatch");
     if (ip < ms->window.base + ms->nextToUpdate) return 0;   /* skipped area */
-    ZSTD_updateDUBT(ms, cParams, ip, iLimit, mls);
-    return ZSTD_DUBT_findBestMatch(ms, cParams, ip, iLimit, offsetPtr, mls, dictMode);
+    ZSTD_updateDUBT(ms, ip, iLimit, mls);
+    return ZSTD_DUBT_findBestMatch(ms, ip, iLimit, offsetPtr, mls, dictMode);
 }
 
 
 static size_t ZSTD_BtFindBestMatch_selectMLS (
-                        ZSTD_matchState_t* ms, ZSTD_compressionParameters const* cParams,
+                        ZSTD_matchState_t* ms,
                         const BYTE* ip, const BYTE* const iLimit,
                         size_t* offsetPtr)
 {
-    switch(cParams->searchLength)
+    switch(ms->cParams.searchLength)
     {
     default : /* includes case 3 */
-    case 4 : return ZSTD_BtFindBestMatch(ms, cParams, ip, iLimit, offsetPtr, 4, ZSTD_noDict);
-    case 5 : return ZSTD_BtFindBestMatch(ms, cParams, ip, iLimit, offsetPtr, 5, ZSTD_noDict);
+    case 4 : return ZSTD_BtFindBestMatch(ms, ip, iLimit, offsetPtr, 4, ZSTD_noDict);
+    case 5 : return ZSTD_BtFindBestMatch(ms, ip, iLimit, offsetPtr, 5, ZSTD_noDict);
     case 7 :
-    case 6 : return ZSTD_BtFindBestMatch(ms, cParams, ip, iLimit, offsetPtr, 6, ZSTD_noDict);
+    case 6 : return ZSTD_BtFindBestMatch(ms, ip, iLimit, offsetPtr, 6, ZSTD_noDict);
     }
 }
 
 
 static size_t ZSTD_BtFindBestMatch_dictMatchState_selectMLS (
-                        ZSTD_matchState_t* ms, ZSTD_compressionParameters const* cParams,
+                        ZSTD_matchState_t* ms,
                         const BYTE* ip, const BYTE* const iLimit,
                         size_t* offsetPtr)
 {
-    switch(cParams->searchLength)
+    switch(ms->cParams.searchLength)
     {
     default : /* includes case 3 */
-    case 4 : return ZSTD_BtFindBestMatch(ms, cParams, ip, iLimit, offsetPtr, 4, ZSTD_dictMatchState);
-    case 5 : return ZSTD_BtFindBestMatch(ms, cParams, ip, iLimit, offsetPtr, 5, ZSTD_dictMatchState);
+    case 4 : return ZSTD_BtFindBestMatch(ms, ip, iLimit, offsetPtr, 4, ZSTD_dictMatchState);
+    case 5 : return ZSTD_BtFindBestMatch(ms, ip, iLimit, offsetPtr, 5, ZSTD_dictMatchState);
     case 7 :
-    case 6 : return ZSTD_BtFindBestMatch(ms, cParams, ip, iLimit, offsetPtr, 6, ZSTD_dictMatchState);
+    case 6 : return ZSTD_BtFindBestMatch(ms, ip, iLimit, offsetPtr, 6, ZSTD_dictMatchState);
     }
 }
 
 
 static size_t ZSTD_BtFindBestMatch_extDict_selectMLS (
-                        ZSTD_matchState_t* ms, ZSTD_compressionParameters const* cParams,
+                        ZSTD_matchState_t* ms,
                         const BYTE* ip, const BYTE* const iLimit,
                         size_t* offsetPtr)
 {
-    switch(cParams->searchLength)
+    switch(ms->cParams.searchLength)
     {
     default : /* includes case 3 */
-    case 4 : return ZSTD_BtFindBestMatch(ms, cParams, ip, iLimit, offsetPtr, 4, ZSTD_extDict);
-    case 5 : return ZSTD_BtFindBestMatch(ms, cParams, ip, iLimit, offsetPtr, 5, ZSTD_extDict);
+    case 4 : return ZSTD_BtFindBestMatch(ms, ip, iLimit, offsetPtr, 4, ZSTD_extDict);
+    case 5 : return ZSTD_BtFindBestMatch(ms, ip, iLimit, offsetPtr, 5, ZSTD_extDict);
     case 7 :
-    case 6 : return ZSTD_BtFindBestMatch(ms, cParams, ip, iLimit, offsetPtr, 6, ZSTD_extDict);
+    case 6 : return ZSTD_BtFindBestMatch(ms, ip, iLimit, offsetPtr, 6, ZSTD_extDict);
     }
 }
 
@@ -458,11 +462,12 @@ U32 ZSTD_insertAndFindFirstIndex(ZSTD_matchState_t* ms, const BYTE* ip) {
 /* inlining is important to hardwire a hot branch (template emulation) */
 FORCE_INLINE_TEMPLATE
 size_t ZSTD_HcFindBestMatch_generic (
-                        ZSTD_matchState_t* ms, ZSTD_compressionParameters const* cParams,
+                        ZSTD_matchState_t* ms,
                         const BYTE* const ip, const BYTE* const iLimit,
                         size_t* offsetPtr,
                         const U32 mls, const ZSTD_dictMode_e dictMode)
 {
+    const ZSTD_compressionParameters* const cParams = &ms->cParams;
     U32* const chainTable = ms->chainTable;
     const U32 chainSize = (1 << cParams->chainLog);
     const U32 chainMask = chainSize-1;
@@ -540,49 +545,49 @@ size_t ZSTD_HcFindBestMatch_generic (
 
 
 FORCE_INLINE_TEMPLATE size_t ZSTD_HcFindBestMatch_selectMLS (
-                        ZSTD_matchState_t* ms, ZSTD_compressionParameters const* cParams,
+                        ZSTD_matchState_t* ms,
                         const BYTE* ip, const BYTE* const iLimit,
                         size_t* offsetPtr)
 {
-    switch(cParams->searchLength)
+    switch(ms->cParams.searchLength)
     {
     default : /* includes case 3 */
-    case 4 : return ZSTD_HcFindBestMatch_generic(ms, cParams, ip, iLimit, offsetPtr, 4, ZSTD_noDict);
-    case 5 : return ZSTD_HcFindBestMatch_generic(ms, cParams, ip, iLimit, offsetPtr, 5, ZSTD_noDict);
+    case 4 : return ZSTD_HcFindBestMatch_generic(ms, ip, iLimit, offsetPtr, 4, ZSTD_noDict);
+    case 5 : return ZSTD_HcFindBestMatch_generic(ms, ip, iLimit, offsetPtr, 5, ZSTD_noDict);
     case 7 :
-    case 6 : return ZSTD_HcFindBestMatch_generic(ms, cParams, ip, iLimit, offsetPtr, 6, ZSTD_noDict);
+    case 6 : return ZSTD_HcFindBestMatch_generic(ms, ip, iLimit, offsetPtr, 6, ZSTD_noDict);
     }
 }
 
 
 static size_t ZSTD_HcFindBestMatch_dictMatchState_selectMLS (
-                        ZSTD_matchState_t* ms, ZSTD_compressionParameters const* cParams,
+                        ZSTD_matchState_t* ms,
                         const BYTE* ip, const BYTE* const iLimit,
                         size_t* offsetPtr)
 {
-    switch(cParams->searchLength)
+    switch(ms->cParams.searchLength)
     {
     default : /* includes case 3 */
-    case 4 : return ZSTD_HcFindBestMatch_generic(ms, cParams, ip, iLimit, offsetPtr, 4, ZSTD_dictMatchState);
-    case 5 : return ZSTD_HcFindBestMatch_generic(ms, cParams, ip, iLimit, offsetPtr, 5, ZSTD_dictMatchState);
+    case 4 : return ZSTD_HcFindBestMatch_generic(ms, ip, iLimit, offsetPtr, 4, ZSTD_dictMatchState);
+    case 5 : return ZSTD_HcFindBestMatch_generic(ms, ip, iLimit, offsetPtr, 5, ZSTD_dictMatchState);
     case 7 :
-    case 6 : return ZSTD_HcFindBestMatch_generic(ms, cParams, ip, iLimit, offsetPtr, 6, ZSTD_dictMatchState);
+    case 6 : return ZSTD_HcFindBestMatch_generic(ms, ip, iLimit, offsetPtr, 6, ZSTD_dictMatchState);
     }
 }
 
 
 FORCE_INLINE_TEMPLATE size_t ZSTD_HcFindBestMatch_extDict_selectMLS (
-                        ZSTD_matchState_t* ms, ZSTD_compressionParameters const* cParams,
+                        ZSTD_matchState_t* ms,
                         const BYTE* ip, const BYTE* const iLimit,
                         size_t* offsetPtr)
 {
-    switch(cParams->searchLength)
+    switch(ms->cParams.searchLength)
     {
     default : /* includes case 3 */
-    case 4 : return ZSTD_HcFindBestMatch_generic(ms, cParams, ip, iLimit, offsetPtr, 4, ZSTD_extDict);
-    case 5 : return ZSTD_HcFindBestMatch_generic(ms, cParams, ip, iLimit, offsetPtr, 5, ZSTD_extDict);
+    case 4 : return ZSTD_HcFindBestMatch_generic(ms, ip, iLimit, offsetPtr, 4, ZSTD_extDict);
+    case 5 : return ZSTD_HcFindBestMatch_generic(ms, ip, iLimit, offsetPtr, 5, ZSTD_extDict);
     case 7 :
-    case 6 : return ZSTD_HcFindBestMatch_generic(ms, cParams, ip, iLimit, offsetPtr, 6, ZSTD_extDict);
+    case 6 : return ZSTD_HcFindBestMatch_generic(ms, ip, iLimit, offsetPtr, 6, ZSTD_extDict);
     }
 }
 
@@ -606,10 +611,9 @@ size_t ZSTD_compressBlock_lazy_generic(
     const BYTE* const base = ms->window.base;
     const U32 prefixLowestIndex = ms->window.dictLimit;
     const BYTE* const prefixLowest = base + prefixLowestIndex;
-    const ZSTD_compressionParameters* cParams = &ms->cParams;
 
     typedef size_t (*searchMax_f)(
-                        ZSTD_matchState_t* ms, ZSTD_compressionParameters const* cParams,
+                        ZSTD_matchState_t* ms,
                         const BYTE* ip, const BYTE* iLimit, size_t* offsetPtr);
     searchMax_f const searchMax = dictMode == ZSTD_dictMatchState ?
         (searchMethod ? ZSTD_BtFindBestMatch_dictMatchState_selectMLS : ZSTD_HcFindBestMatch_dictMatchState_selectMLS) :
@@ -630,8 +634,6 @@ size_t ZSTD_compressBlock_lazy_generic(
                                      0;
     const U32 dictAndPrefixLength = (U32)(ip - prefixLowest + dictEnd - dictLowest);
 
-    (void)dictMode;
-
     /* init */
     ip += (dictAndPrefixLength == 0);
     ms->nextToUpdate3 = ms->nextToUpdate;
@@ -675,7 +677,7 @@ size_t ZSTD_compressBlock_lazy_generic(
 
         /* first search (depth 0) */
         {   size_t offsetFound = 99999999;
-            size_t const ml2 = searchMax(ms, cParams, ip, iend, &offsetFound);
+            size_t const ml2 = searchMax(ms, ip, iend, &offsetFound);
             if (ml2 > matchLength)
                 matchLength = ml2, start = ip, offset=offsetFound;
         }
@@ -713,7 +715,7 @@ size_t ZSTD_compressBlock_lazy_generic(
                 }
             }
             {   size_t offset2=99999999;
-                size_t const ml2 = searchMax(ms, cParams, ip, iend, &offset2);
+                size_t const ml2 = searchMax(ms, ip, iend, &offset2);
                 int const gain2 = (int)(ml2*4 - ZSTD_highbit32((U32)offset2+1));   /* raw approx */
                 int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offset+1) + 4);
                 if ((ml2 >= 4) && (gain2 > gain1)) {
@@ -748,7 +750,7 @@ size_t ZSTD_compressBlock_lazy_generic(
                     }
                 }
                 {   size_t offset2=99999999;
-                    size_t const ml2 = searchMax(ms, cParams, ip, iend, &offset2);
+                    size_t const ml2 = searchMax(ms, ip, iend, &offset2);
                     int const gain2 = (int)(ml2*4 - ZSTD_highbit32((U32)offset2+1));   /* raw approx */
                     int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offset+1) + 7);
                     if ((ml2 >= 4) && (gain2 > gain1)) {
@@ -905,10 +907,9 @@ size_t ZSTD_compressBlock_lazy_extDict_generic(
     const BYTE* const dictBase = ms->window.dictBase;
     const BYTE* const dictEnd  = dictBase + dictLimit;
     const BYTE* const dictStart  = dictBase + lowestIndex;
-    const ZSTD_compressionParameters* const cParams = &ms->cParams;
 
     typedef size_t (*searchMax_f)(
-                        ZSTD_matchState_t* ms, ZSTD_compressionParameters const* cParams,
+                        ZSTD_matchState_t* ms,
                         const BYTE* ip, const BYTE* iLimit, size_t* offsetPtr);
     searchMax_f searchMax = searchMethod ? ZSTD_BtFindBestMatch_extDict_selectMLS : ZSTD_HcFindBestMatch_extDict_selectMLS;
 
@@ -939,7 +940,7 @@ size_t ZSTD_compressBlock_lazy_extDict_generic(
 
         /* first search (depth 0) */
         {   size_t offsetFound = 99999999;
-            size_t const ml2 = searchMax(ms, cParams, ip, iend, &offsetFound);
+            size_t const ml2 = searchMax(ms, ip, iend, &offsetFound);
             if (ml2 > matchLength)
                 matchLength = ml2, start = ip, offset=offsetFound;
         }
@@ -972,7 +973,7 @@ size_t ZSTD_compressBlock_lazy_extDict_generic(
 
             /* search match, depth 1 */
             {   size_t offset2=99999999;
-                size_t const ml2 = searchMax(ms, cParams, ip, iend, &offset2);
+                size_t const ml2 = searchMax(ms, ip, iend, &offset2);
                 int const gain2 = (int)(ml2*4 - ZSTD_highbit32((U32)offset2+1));   /* raw approx */
                 int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offset+1) + 4);
                 if ((ml2 >= 4) && (gain2 > gain1)) {
@@ -1002,7 +1003,7 @@ size_t ZSTD_compressBlock_lazy_extDict_generic(
 
                 /* search match, depth 2 */
                 {   size_t offset2=99999999;
-                    size_t const ml2 = searchMax(ms, cParams, ip, iend, &offset2);
+                    size_t const ml2 = searchMax(ms, ip, iend, &offset2);
                     int const gain2 = (int)(ml2*4 - ZSTD_highbit32((U32)offset2+1));   /* raw approx */
                     int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offset+1) + 7);
                     if ((ml2 >= 4) && (gain2 > gain1)) {

From 50cc1cf4d53d8a6bf75ba7084bb30f4f7abece16 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" 
Date: Thu, 23 Aug 2018 12:21:27 -0700
Subject: [PATCH 333/372] Remove CParams Arg from ZSTD_ldm_blockCompress

---
 lib/compress/zstd_compress.c | 2 --
 lib/compress/zstd_ldm.c      | 9 ++++-----
 lib/compress/zstd_ldm.h      | 1 -
 3 files changed, 4 insertions(+), 8 deletions(-)

diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index 633400f7e..6c31d2254 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -2394,7 +2394,6 @@ static size_t ZSTD_compressBlock_internal(ZSTD_CCtx* zc,
                 ZSTD_ldm_blockCompress(&zc->externSeqStore,
                                        ms, &zc->seqStore,
                                        zc->blockState.nextCBlock->rep,
-                                       &zc->appliedParams.cParams,
                                        src, srcSize);
             assert(zc->externSeqStore.pos <= zc->externSeqStore.size);
         } else if (zc->appliedParams.ldmParams.enableLdm) {
@@ -2411,7 +2410,6 @@ static size_t ZSTD_compressBlock_internal(ZSTD_CCtx* zc,
                 ZSTD_ldm_blockCompress(&ldmSeqStore,
                                        ms, &zc->seqStore,
                                        zc->blockState.nextCBlock->rep,
-                                       &zc->appliedParams.cParams,
                                        src, srcSize);
             assert(ldmSeqStore.pos == ldmSeqStore.size);
         } else {   /* not long range mode */
diff --git a/lib/compress/zstd_ldm.c b/lib/compress/zstd_ldm.c
index 99cbff980..6238ddecf 100644
--- a/lib/compress/zstd_ldm.c
+++ b/lib/compress/zstd_ldm.c
@@ -590,8 +590,9 @@ static rawSeq maybeSplitSequence(rawSeqStore_t* rawSeqStore,
 
 size_t ZSTD_ldm_blockCompress(rawSeqStore_t* rawSeqStore,
     ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
-    ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize)
+    void const* src, size_t srcSize)
 {
+    const ZSTD_compressionParameters* const cParams = &ms->cParams;
     unsigned const minMatch = cParams->searchLength;
     ZSTD_blockCompressor const blockCompressor =
         ZSTD_selectBlockCompressor(cParams->strategy, ZSTD_matchState_dictMode(ms));
@@ -624,8 +625,7 @@ size_t ZSTD_ldm_blockCompress(rawSeqStore_t* rawSeqStore,
         DEBUGLOG(5, "calling block compressor on segment of size %u", sequence.litLength);
         {
             size_t const newLitLength =
-                blockCompressor(ms, seqStore, rep, ip,
-                                sequence.litLength);
+                blockCompressor(ms, seqStore, rep, ip, sequence.litLength);
             ip += sequence.litLength;
             /* Update the repcodes */
             for (i = ZSTD_REP_NUM - 1; i > 0; i--)
@@ -642,6 +642,5 @@ size_t ZSTD_ldm_blockCompress(rawSeqStore_t* rawSeqStore,
     ZSTD_ldm_limitTableUpdate(ms, ip);
     ZSTD_ldm_fillFastTables(ms, ip);
     /* Compress the last literals */
-    return blockCompressor(ms, seqStore, rep,
-                           ip, iend - ip);
+    return blockCompressor(ms, seqStore, rep, ip, iend - ip);
 }
diff --git a/lib/compress/zstd_ldm.h b/lib/compress/zstd_ldm.h
index 96588adb0..21fba4d59 100644
--- a/lib/compress/zstd_ldm.h
+++ b/lib/compress/zstd_ldm.h
@@ -61,7 +61,6 @@ size_t ZSTD_ldm_generateSequences(
  */
 size_t ZSTD_ldm_blockCompress(rawSeqStore_t* rawSeqStore,
             ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
-            ZSTD_compressionParameters const* cParams,
             void const* src, size_t srcSize);
 
 /**

From 01e34d365b1efaf569f059c948de48083d145562 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" 
Date: Thu, 23 Aug 2018 12:35:03 -0700
Subject: [PATCH 334/372] Strengthen Assertion to Assert Equality

---
 lib/compress/zstd_compress.c | 16 ++++++++++++++--
 1 file changed, 14 insertions(+), 2 deletions(-)

diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index 6c31d2254..d4a159039 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -936,6 +936,18 @@ static U32 ZSTD_equivalentCParams(ZSTD_compressionParameters cParams1,
          & ((cParams1.searchLength==3) == (cParams2.searchLength==3));  /* hashlog3 space */
 }
 
+static U32 ZSTD_equalCParams(ZSTD_compressionParameters cParams1,
+                             ZSTD_compressionParameters cParams2)
+{
+    return (cParams1.windowLog    == cParams2.windowLog)
+         & (cParams1.chainLog     == cParams2.chainLog)
+         & (cParams1.hashLog      == cParams2.hashLog)
+         & (cParams1.searchLog    == cParams2.searchLog)
+         & (cParams1.searchLength == cParams2.searchLength)
+         & (cParams1.targetLength == cParams2.targetLength)
+         & (cParams1.strategy     == cParams2.strategy);
+}
+
 /** The parameters are equivalent if ldm is not enabled in both sets or
  *  all the parameters are equivalent. */
 static U32 ZSTD_equivalentLdmParams(ldmParams_t ldmParams1,
@@ -2357,7 +2369,7 @@ static size_t ZSTD_compressBlock_internal(ZSTD_CCtx* zc,
     assert(srcSize <= ZSTD_BLOCKSIZE_MAX);
 
     /* Assert that we have correctly flushed the ctx params into the ms's copy */
-    assert(ZSTD_equivalentCParams(zc->appliedParams.cParams, ms->cParams));
+    assert(ZSTD_equalCParams(zc->appliedParams.cParams, ms->cParams));
 
     if (srcSize < MIN_CBLOCK_SIZE+ZSTD_blockHeaderSize+1) {
         ZSTD_ldm_skipSequences(&zc->externSeqStore, srcSize, zc->appliedParams.cParams.searchLength);
@@ -2698,7 +2710,7 @@ static size_t ZSTD_loadDictionaryContent(ZSTD_matchState_t* ms,
     ms->loadedDictEnd = params->forceWindow ? 0 : (U32)(iend - ms->window.base);
 
     /* Assert that we the ms params match the params we're being given */
-    assert(ZSTD_equivalentCParams(params->cParams, ms->cParams));
+    assert(ZSTD_equalCParams(params->cParams, ms->cParams));
 
     if (srcSize <= HASH_READ_SIZE) return 0;
 

From 7212b5e5c2a1bfdc009202b9a5c0f6fc01c2d96b Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" 
Date: Thu, 23 Aug 2018 12:53:16 -0700
Subject: [PATCH 335/372] Move Match State CParams Setting into `resetCCtx` and
 `continueCCtx`

---
 lib/compress/zstd_compress.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index d4a159039..3e17e40b3 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -1049,6 +1049,7 @@ static size_t ZSTD_continueCCtx(ZSTD_CCtx* cctx, ZSTD_CCtx_params params, U64 pl
 
     cctx->blockSize = blockSize;   /* previous block size could be different even for same windowLog, due to pledgedSrcSize */
     cctx->appliedParams = params;
+    cctx->blockState.matchState.cParams = params.cParams;
     cctx->pledgedSrcSizePlusOne = pledgedSrcSize+1;
     cctx->consumedSrcSize = 0;
     cctx->producedCSize = 0;
@@ -1217,6 +1218,7 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc,
 
         /* init params */
         zc->appliedParams = params;
+        zc->blockState.matchState.cParams = params.cParams;
         zc->pledgedSrcSizePlusOne = pledgedSrcSize+1;
         zc->consumedSrcSize = 0;
         zc->producedCSize = 0;
@@ -1390,7 +1392,6 @@ static size_t ZSTD_resetCCtx_usingCDict(ZSTD_CCtx* cctx,
             dstMatchState->nextToUpdate = srcMatchState->nextToUpdate;
             dstMatchState->nextToUpdate3= srcMatchState->nextToUpdate3;
             dstMatchState->loadedDictEnd= srcMatchState->loadedDictEnd;
-            dstMatchState->cParams      = srcMatchState->cParams;
         }
     }
 

From 1f188ae655d1248c43a852d2a5a3bd608c20e42c Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" 
Date: Thu, 23 Aug 2018 14:09:18 -0700
Subject: [PATCH 336/372] Move Asserts into Function to Avoid Unused Function
 Warning

---
 lib/compress/zstd_compress.c | 22 +++++++++++-----------
 1 file changed, 11 insertions(+), 11 deletions(-)

diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index 3e17e40b3..c89b6e88c 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -936,16 +936,16 @@ static U32 ZSTD_equivalentCParams(ZSTD_compressionParameters cParams1,
          & ((cParams1.searchLength==3) == (cParams2.searchLength==3));  /* hashlog3 space */
 }
 
-static U32 ZSTD_equalCParams(ZSTD_compressionParameters cParams1,
-                             ZSTD_compressionParameters cParams2)
+static void ZSTD_assertEqualCParams(ZSTD_compressionParameters cParams1,
+                                    ZSTD_compressionParameters cParams2)
 {
-    return (cParams1.windowLog    == cParams2.windowLog)
-         & (cParams1.chainLog     == cParams2.chainLog)
-         & (cParams1.hashLog      == cParams2.hashLog)
-         & (cParams1.searchLog    == cParams2.searchLog)
-         & (cParams1.searchLength == cParams2.searchLength)
-         & (cParams1.targetLength == cParams2.targetLength)
-         & (cParams1.strategy     == cParams2.strategy);
+    assert(cParams1.windowLog    == cParams2.windowLog);
+    assert(cParams1.chainLog     == cParams2.chainLog);
+    assert(cParams1.hashLog      == cParams2.hashLog);
+    assert(cParams1.searchLog    == cParams2.searchLog);
+    assert(cParams1.searchLength == cParams2.searchLength);
+    assert(cParams1.targetLength == cParams2.targetLength);
+    assert(cParams1.strategy     == cParams2.strategy);
 }
 
 /** The parameters are equivalent if ldm is not enabled in both sets or
@@ -2370,7 +2370,7 @@ static size_t ZSTD_compressBlock_internal(ZSTD_CCtx* zc,
     assert(srcSize <= ZSTD_BLOCKSIZE_MAX);
 
     /* Assert that we have correctly flushed the ctx params into the ms's copy */
-    assert(ZSTD_equalCParams(zc->appliedParams.cParams, ms->cParams));
+    ZSTD_assertEqualCParams(zc->appliedParams.cParams, ms->cParams);
 
     if (srcSize < MIN_CBLOCK_SIZE+ZSTD_blockHeaderSize+1) {
         ZSTD_ldm_skipSequences(&zc->externSeqStore, srcSize, zc->appliedParams.cParams.searchLength);
@@ -2711,7 +2711,7 @@ static size_t ZSTD_loadDictionaryContent(ZSTD_matchState_t* ms,
     ms->loadedDictEnd = params->forceWindow ? 0 : (U32)(iend - ms->window.base);
 
     /* Assert that we the ms params match the params we're being given */
-    assert(ZSTD_equalCParams(params->cParams, ms->cParams));
+    ZSTD_assertEqualCParams(params->cParams, ms->cParams);
 
     if (srcSize <= HASH_READ_SIZE) return 0;
 

From b7fba599ae04b2d9553417740f3542df0cdd66aa Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" 
Date: Thu, 23 Aug 2018 14:36:51 -0700
Subject: [PATCH 337/372] And Then Avoid the Unused Parameter Warning

---
 lib/compress/zstd_compress.c | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index c89b6e88c..2a279edec 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -939,6 +939,8 @@ static U32 ZSTD_equivalentCParams(ZSTD_compressionParameters cParams1,
 static void ZSTD_assertEqualCParams(ZSTD_compressionParameters cParams1,
                                     ZSTD_compressionParameters cParams2)
 {
+    (void)cParams1;
+    (void)cParams2;
     assert(cParams1.windowLog    == cParams2.windowLog);
     assert(cParams1.chainLog     == cParams2.chainLog);
     assert(cParams1.hashLog      == cParams2.hashLog);

From a6d6bbeae1e2b47f258d085cd44712515671d160 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" 
Date: Fri, 24 Aug 2018 13:13:48 -0700
Subject: [PATCH 338/372] Pull Attachment Decision into Separate Function

---
 lib/compress/zstd_compress.c | 52 +++++++++++++++++++++++-------------
 1 file changed, 33 insertions(+), 19 deletions(-)

diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index 2a279edec..591916dbb 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -1302,6 +1302,38 @@ void ZSTD_invalidateRepCodes(ZSTD_CCtx* cctx) {
     assert(!ZSTD_window_hasExtDict(cctx->blockState.matchState.window));
 }
 
+/* These are the approximate sizes for each strategy past which copying the
+ * dictionary tables into the working context is faster than using them
+ * in-place.
+ */
+static const size_t attachDictSizeCutoffs[(unsigned)ZSTD_btultra+1] = {
+    8 KB, /* unused */
+    8 KB, /* ZSTD_fast */
+    16 KB, /* ZSTD_dfast */
+    32 KB, /* ZSTD_greedy */
+    32 KB, /* ZSTD_lazy */
+    32 KB, /* ZSTD_lazy2 */
+    32 KB, /* ZSTD_btlazy2 */
+    32 KB, /* ZSTD_btopt */
+    8 KB /* ZSTD_btultra */
+};
+
+static int ZSTD_shouldAttachDict(ZSTD_CCtx* cctx,
+                            const ZSTD_CDict* cdict,
+                            ZSTD_CCtx_params params,
+                            U64 pledgedSrcSize)
+{
+    size_t cutoff = attachDictSizeCutoffs[cdict->matchState.cParams.strategy];
+    return ( pledgedSrcSize <= cutoff
+          || pledgedSrcSize == ZSTD_CONTENTSIZE_UNKNOWN
+          || params.attachDictPref == ZSTD_dictForceAttach )
+        && params.attachDictPref != ZSTD_dictForceCopy
+        && !params.forceWindow /* dictMatchState isn't correctly
+                                * handled in _enforceMaxDist */
+        && ZSTD_equivalentCParams(cctx->appliedParams.cParams,
+                                  cdict->matchState.cParams);
+}
+
 static size_t ZSTD_resetCCtx_usingCDict(ZSTD_CCtx* cctx,
                             const ZSTD_CDict* cdict,
                             ZSTD_CCtx_params params,
@@ -1311,26 +1343,8 @@ static size_t ZSTD_resetCCtx_usingCDict(ZSTD_CCtx* cctx,
     /* We have a choice between copying the dictionary context into the working
      * context, or referencing the dictionary context from the working context
      * in-place. We decide here which strategy to use. */
-    const U64 attachDictSizeCutoffs[(unsigned)ZSTD_btultra+1] = {
-        8 KB, /* unused */
-        8 KB, /* ZSTD_fast */
-        16 KB, /* ZSTD_dfast */
-        32 KB, /* ZSTD_greedy */
-        32 KB, /* ZSTD_lazy */
-        32 KB, /* ZSTD_lazy2 */
-        32 KB, /* ZSTD_btlazy2 */
-        32 KB, /* ZSTD_btopt */
-        8 KB /* ZSTD_btultra */
-    };
     const ZSTD_compressionParameters *cdict_cParams = &cdict->matchState.cParams;
-    const int attachDict = ( pledgedSrcSize <= attachDictSizeCutoffs[cdict_cParams->strategy]
-                          || pledgedSrcSize == ZSTD_CONTENTSIZE_UNKNOWN
-                          || params.attachDictPref == ZSTD_dictForceAttach )
-                        && params.attachDictPref != ZSTD_dictForceCopy
-                        && !params.forceWindow /* dictMatchState isn't correctly
-                                                * handled in _enforceMaxDist */
-                        && ZSTD_equivalentCParams(cctx->appliedParams.cParams,
-                                                  *cdict_cParams);
+    const int attachDict = ZSTD_shouldAttachDict(cctx, cdict, params, pledgedSrcSize);
 
     DEBUGLOG(4, "ZSTD_resetCCtx_usingCDict (pledgedSrcSize=%u)", (U32)pledgedSrcSize);
 

From 01ff945eae07744e383485d938d333ca3c973003 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" 
Date: Fri, 24 Aug 2018 15:54:58 -0700
Subject: [PATCH 339/372] Split Attach and Copy Reset Strategies into Separate
 Implementation Functions

---
 lib/compress/zstd_compress.c | 124 +++++++++++++++++++++++------------
 1 file changed, 83 insertions(+), 41 deletions(-)

diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index 591916dbb..174f1791d 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -1334,35 +1334,27 @@ static int ZSTD_shouldAttachDict(ZSTD_CCtx* cctx,
                                   cdict->matchState.cParams);
 }
 
-static size_t ZSTD_resetCCtx_usingCDict(ZSTD_CCtx* cctx,
+static size_t ZSTD_resetCCtx_byAttachingCDict(ZSTD_CCtx* cctx,
                             const ZSTD_CDict* cdict,
                             ZSTD_CCtx_params params,
                             U64 pledgedSrcSize,
                             ZSTD_buffered_policy_e zbuff)
 {
-    /* We have a choice between copying the dictionary context into the working
-     * context, or referencing the dictionary context from the working context
-     * in-place. We decide here which strategy to use. */
-    const ZSTD_compressionParameters *cdict_cParams = &cdict->matchState.cParams;
-    const int attachDict = ZSTD_shouldAttachDict(cctx, cdict, params, pledgedSrcSize);
-
-    DEBUGLOG(4, "ZSTD_resetCCtx_usingCDict (pledgedSrcSize=%u)", (U32)pledgedSrcSize);
-
-
-    {   unsigned const windowLog = params.cParams.windowLog;
+    {
+        const ZSTD_compressionParameters *cdict_cParams = &cdict->matchState.cParams;
+        unsigned const windowLog = params.cParams.windowLog;
         assert(windowLog != 0);
         /* Copy only compression parameters related to tables. */
         params.cParams = *cdict_cParams;
         params.cParams.windowLog = windowLog;
         ZSTD_resetCCtx_internal(cctx, params, pledgedSrcSize,
-                                attachDict ? ZSTDcrp_continue : ZSTDcrp_noMemset,
-                                zbuff);
+                                ZSTDcrp_continue, zbuff);
         assert(cctx->appliedParams.cParams.strategy == cdict_cParams->strategy);
         assert(cctx->appliedParams.cParams.hashLog == cdict_cParams->hashLog);
         assert(cctx->appliedParams.cParams.chainLog == cdict_cParams->chainLog);
     }
 
-    if (attachDict) {
+    {
         const U32 cdictEnd = (U32)( cdict->matchState.window.nextSrc
                                   - cdict->matchState.window.base);
         const U32 cdictLen = cdictEnd - cdict->matchState.window.dictLimit;
@@ -1382,33 +1374,6 @@ static size_t ZSTD_resetCCtx_usingCDict(ZSTD_CCtx* cctx,
             }
             cctx->blockState.matchState.loadedDictEnd = cctx->blockState.matchState.window.dictLimit;
         }
-    } else {
-        DEBUGLOG(4, "copying dictionary into context");
-        /* copy tables */
-        {   size_t const chainSize = (cdict_cParams->strategy == ZSTD_fast) ? 0 : ((size_t)1 << cdict_cParams->chainLog);
-            size_t const hSize =  (size_t)1 << cdict_cParams->hashLog;
-            size_t const tableSpace = (chainSize + hSize) * sizeof(U32);
-            assert((U32*)cctx->blockState.matchState.chainTable == (U32*)cctx->blockState.matchState.hashTable + hSize);  /* chainTable must follow hashTable */
-            assert((U32*)cctx->blockState.matchState.hashTable3 == (U32*)cctx->blockState.matchState.chainTable + chainSize);
-            assert((U32*)cdict->matchState.chainTable == (U32*)cdict->matchState.hashTable + hSize);  /* chainTable must follow hashTable */
-            assert((U32*)cdict->matchState.hashTable3 == (U32*)cdict->matchState.chainTable + chainSize);
-            memcpy(cctx->blockState.matchState.hashTable, cdict->matchState.hashTable, tableSpace);   /* presumes all tables follow each other */
-        }
-
-        /* Zero the hashTable3, since the cdict never fills it */
-        {   size_t const h3Size = (size_t)1 << cctx->blockState.matchState.hashLog3;
-            assert(cdict->matchState.hashLog3 == 0);
-            memset(cctx->blockState.matchState.hashTable3, 0, h3Size * sizeof(U32));
-        }
-
-        /* copy dictionary offsets */
-        {   ZSTD_matchState_t const* srcMatchState = &cdict->matchState;
-            ZSTD_matchState_t* dstMatchState = &cctx->blockState.matchState;
-            dstMatchState->window       = srcMatchState->window;
-            dstMatchState->nextToUpdate = srcMatchState->nextToUpdate;
-            dstMatchState->nextToUpdate3= srcMatchState->nextToUpdate3;
-            dstMatchState->loadedDictEnd= srcMatchState->loadedDictEnd;
-        }
     }
 
     cctx->dictID = cdict->dictID;
@@ -1419,6 +1384,83 @@ static size_t ZSTD_resetCCtx_usingCDict(ZSTD_CCtx* cctx,
     return 0;
 }
 
+static size_t ZSTD_resetCCtx_byCopyingCDict(ZSTD_CCtx* cctx,
+                            const ZSTD_CDict* cdict,
+                            ZSTD_CCtx_params params,
+                            U64 pledgedSrcSize,
+                            ZSTD_buffered_policy_e zbuff)
+{
+    const ZSTD_compressionParameters *cdict_cParams = &cdict->matchState.cParams;
+
+    DEBUGLOG(4, "copying dictionary into context");
+
+    {   unsigned const windowLog = params.cParams.windowLog;
+        assert(windowLog != 0);
+        /* Copy only compression parameters related to tables. */
+        params.cParams = *cdict_cParams;
+        params.cParams.windowLog = windowLog;
+        ZSTD_resetCCtx_internal(cctx, params, pledgedSrcSize,
+                                ZSTDcrp_noMemset, zbuff);
+        assert(cctx->appliedParams.cParams.strategy == cdict_cParams->strategy);
+        assert(cctx->appliedParams.cParams.hashLog == cdict_cParams->hashLog);
+        assert(cctx->appliedParams.cParams.chainLog == cdict_cParams->chainLog);
+    }
+
+    /* copy tables */
+    {   size_t const chainSize = (cdict_cParams->strategy == ZSTD_fast) ? 0 : ((size_t)1 << cdict_cParams->chainLog);
+        size_t const hSize =  (size_t)1 << cdict_cParams->hashLog;
+        size_t const tableSpace = (chainSize + hSize) * sizeof(U32);
+        assert((U32*)cctx->blockState.matchState.chainTable == (U32*)cctx->blockState.matchState.hashTable + hSize);  /* chainTable must follow hashTable */
+        assert((U32*)cctx->blockState.matchState.hashTable3 == (U32*)cctx->blockState.matchState.chainTable + chainSize);
+        assert((U32*)cdict->matchState.chainTable == (U32*)cdict->matchState.hashTable + hSize);  /* chainTable must follow hashTable */
+        assert((U32*)cdict->matchState.hashTable3 == (U32*)cdict->matchState.chainTable + chainSize);
+        memcpy(cctx->blockState.matchState.hashTable, cdict->matchState.hashTable, tableSpace);   /* presumes all tables follow each other */
+    }
+
+    /* Zero the hashTable3, since the cdict never fills it */
+    {   size_t const h3Size = (size_t)1 << cctx->blockState.matchState.hashLog3;
+        assert(cdict->matchState.hashLog3 == 0);
+        memset(cctx->blockState.matchState.hashTable3, 0, h3Size * sizeof(U32));
+    }
+
+    /* copy dictionary offsets */
+    {   ZSTD_matchState_t const* srcMatchState = &cdict->matchState;
+        ZSTD_matchState_t* dstMatchState = &cctx->blockState.matchState;
+        dstMatchState->window       = srcMatchState->window;
+        dstMatchState->nextToUpdate = srcMatchState->nextToUpdate;
+        dstMatchState->nextToUpdate3= srcMatchState->nextToUpdate3;
+        dstMatchState->loadedDictEnd= srcMatchState->loadedDictEnd;
+    }
+
+    cctx->dictID = cdict->dictID;
+
+    /* copy block state */
+    memcpy(cctx->blockState.prevCBlock, &cdict->cBlockState, sizeof(cdict->cBlockState));
+
+    return 0;
+}
+
+/* We have a choice between copying the dictionary context into the working
+ * context, or referencing the dictionary context from the working context
+ * in-place. We decide here which strategy to use. */
+static size_t ZSTD_resetCCtx_usingCDict(ZSTD_CCtx* cctx,
+                            const ZSTD_CDict* cdict,
+                            ZSTD_CCtx_params params,
+                            U64 pledgedSrcSize,
+                            ZSTD_buffered_policy_e zbuff)
+{
+
+    DEBUGLOG(4, "ZSTD_resetCCtx_usingCDict (pledgedSrcSize=%u)", (U32)pledgedSrcSize);
+
+    if (ZSTD_shouldAttachDict(cctx, cdict, params, pledgedSrcSize)) {
+        return ZSTD_resetCCtx_byAttachingCDict(
+            cctx, cdict, params, pledgedSrcSize, zbuff);
+    } else {
+        return ZSTD_resetCCtx_byCopyingCDict(
+            cctx, cdict, params, pledgedSrcSize, zbuff);
+    }
+}
+
 /*! ZSTD_copyCCtx_internal() :
  *  Duplicate an existing context `srcCCtx` into another one `dstCCtx`.
  *  Only works during stage ZSTDcs_init (i.e. after creation, but before first call to ZSTD_compressContinue()).

From eae8232f50bc7c73bef5fa6074e0781264989bbc Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" 
Date: Mon, 27 Aug 2018 13:21:01 -0700
Subject: [PATCH 340/372] For Supported Strategies, Attach Dict Even When
 Params Don't Match

---
 lib/compress/zstd_compress.c | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index 174f1791d..ff49b0c5e 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -1330,8 +1330,10 @@ static int ZSTD_shouldAttachDict(ZSTD_CCtx* cctx,
         && params.attachDictPref != ZSTD_dictForceCopy
         && !params.forceWindow /* dictMatchState isn't correctly
                                 * handled in _enforceMaxDist */
-        && ZSTD_equivalentCParams(cctx->appliedParams.cParams,
-                                  cdict->matchState.cParams);
+        && ( (cdict->matchState.cParams.strategy <= ZSTD_fast)
+          || (cdict->matchState.cParams.strategy > ZSTD_fast &&
+              ZSTD_equivalentCParams(cctx->appliedParams.cParams,
+                                     cdict->matchState.cParams)));
 }
 
 static size_t ZSTD_resetCCtx_byAttachingCDict(ZSTD_CCtx* cctx,

From 34e0193129cfe2195e4b8b34b9d27ba4de955394 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" 
Date: Mon, 27 Aug 2018 15:09:25 -0700
Subject: [PATCH 341/372] Allow Setting Hash and Chain Logs on Contexts with
 Attached CDict

---
 lib/compress/zstd_compress.c | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index ff49b0c5e..e133b37da 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -284,9 +284,11 @@ size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, unsigned v
         if (cctx->cdict) return ERROR(stage_wrong);
         return ZSTD_CCtxParam_setParameter(&cctx->requestedParams, param, value);
 
-    case ZSTD_p_windowLog:
     case ZSTD_p_hashLog:
     case ZSTD_p_chainLog:
+        return ZSTD_CCtxParam_setParameter(&cctx->requestedParams, param, value);
+
+    case ZSTD_p_windowLog:
     case ZSTD_p_searchLog:
     case ZSTD_p_minMatch:
     case ZSTD_p_targetLength:

From b3107c77995db22d7d741cae02cdd03fb6a123fb Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" 
Date: Mon, 27 Aug 2018 15:09:56 -0700
Subject: [PATCH 342/372] Temporary Commit to Retain Requested Hash and Chain
 Logs During Dict Attach

---
 lib/compress/zstd_compress.c | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index e133b37da..4d54772a0 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -1351,11 +1351,21 @@ static size_t ZSTD_resetCCtx_byAttachingCDict(ZSTD_CCtx* cctx,
         /* Copy only compression parameters related to tables. */
         params.cParams = *cdict_cParams;
         params.cParams.windowLog = windowLog;
+        if (params.cParams.strategy <= ZSTD_fast) {
+            DEBUGLOG(4, "Overriding hashLog from %d to %d", params.cParams.hashLog, cctx->requestedParams.cParams.hashLog);
+            DEBUGLOG(4, "Overriding chainLog from %d to %d", params.cParams.chainLog, cctx->requestedParams.cParams.chainLog);
+            if (cctx->requestedParams.cParams.hashLog)
+                params.cParams.hashLog = cctx->requestedParams.cParams.hashLog;
+            if (cctx->requestedParams.cParams.chainLog)
+                params.cParams.chainLog = cctx->requestedParams.cParams.chainLog;
+        }
         ZSTD_resetCCtx_internal(cctx, params, pledgedSrcSize,
                                 ZSTDcrp_continue, zbuff);
         assert(cctx->appliedParams.cParams.strategy == cdict_cParams->strategy);
-        assert(cctx->appliedParams.cParams.hashLog == cdict_cParams->hashLog);
-        assert(cctx->appliedParams.cParams.chainLog == cdict_cParams->chainLog);
+        if (params.cParams.strategy > ZSTD_fast) {
+            assert(cctx->appliedParams.cParams.hashLog == cdict_cParams->hashLog);
+            assert(cctx->appliedParams.cParams.chainLog == cdict_cParams->chainLog);
+        }
     }
 
     {

From bc880ebe8f315cb8b7c1a61745c562ddfa6dfbec Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" 
Date: Thu, 23 Aug 2018 14:21:35 -0700
Subject: [PATCH 343/372] Stop Passing in `hashLog` and `stepSize` to
 `ZSTD_compressBlock_fast_generic`

---
 lib/compress/zstd_fast.c | 49 +++++++++++++++++++---------------------
 1 file changed, 23 insertions(+), 26 deletions(-)

diff --git a/lib/compress/zstd_fast.c b/lib/compress/zstd_fast.c
index 28961db06..bb7234846 100644
--- a/lib/compress/zstd_fast.c
+++ b/lib/compress/zstd_fast.c
@@ -45,10 +45,13 @@ FORCE_INLINE_TEMPLATE
 size_t ZSTD_compressBlock_fast_generic(
         ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
         void const* src, size_t srcSize,
-        U32 const hlog, U32 stepSize, U32 const mls,
-        ZSTD_dictMode_e const dictMode)
+        U32 const mls, ZSTD_dictMode_e const dictMode)
 {
+    const ZSTD_compressionParameters* const cParams = &ms->cParams;
     U32* const hashTable = ms->hashTable;
+    U32 const hlog = cParams->hashLog;
+    /* support stepSize of 0 */
+    U32 const stepSize = cParams->targetLength + !(cParams->targetLength);
     const BYTE* const base = ms->window.base;
     const BYTE* const istart = (const BYTE*)src;
     const BYTE* ip = istart;
@@ -84,7 +87,6 @@ size_t ZSTD_compressBlock_fast_generic(
         || prefixStartIndex >= (U32)(dictEnd - dictBase));
 
     /* init */
-    stepSize += !stepSize;  /* support stepSize of 0 */
     ip += (dictAndPrefixLength == 0);
     if (dictMode == ZSTD_noDict) {
         U32 const maxRep = (U32)(ip - prefixStart);
@@ -227,21 +229,19 @@ size_t ZSTD_compressBlock_fast(
         void const* src, size_t srcSize)
 {
     ZSTD_compressionParameters const* cParams = &ms->cParams;
-    U32 const hlog = cParams->hashLog;
     U32 const mls = cParams->searchLength;
-    U32 const stepSize = cParams->targetLength;
     assert(ms->dictMatchState == NULL);
     switch(mls)
     {
     default: /* includes case 3 */
     case 4 :
-        return ZSTD_compressBlock_fast_generic(ms, seqStore, rep, src, srcSize, hlog, stepSize, 4, ZSTD_noDict);
+        return ZSTD_compressBlock_fast_generic(ms, seqStore, rep, src, srcSize, 4, ZSTD_noDict);
     case 5 :
-        return ZSTD_compressBlock_fast_generic(ms, seqStore, rep, src, srcSize, hlog, stepSize, 5, ZSTD_noDict);
+        return ZSTD_compressBlock_fast_generic(ms, seqStore, rep, src, srcSize, 5, ZSTD_noDict);
     case 6 :
-        return ZSTD_compressBlock_fast_generic(ms, seqStore, rep, src, srcSize, hlog, stepSize, 6, ZSTD_noDict);
+        return ZSTD_compressBlock_fast_generic(ms, seqStore, rep, src, srcSize, 6, ZSTD_noDict);
     case 7 :
-        return ZSTD_compressBlock_fast_generic(ms, seqStore, rep, src, srcSize, hlog, stepSize, 7, ZSTD_noDict);
+        return ZSTD_compressBlock_fast_generic(ms, seqStore, rep, src, srcSize, 7, ZSTD_noDict);
     }
 }
 
@@ -250,31 +250,32 @@ size_t ZSTD_compressBlock_fast_dictMatchState(
         void const* src, size_t srcSize)
 {
     ZSTD_compressionParameters const* cParams = &ms->cParams;
-    U32 const hlog = cParams->hashLog;
     U32 const mls = cParams->searchLength;
-    U32 const stepSize = cParams->targetLength;
     assert(ms->dictMatchState != NULL);
     switch(mls)
     {
     default: /* includes case 3 */
     case 4 :
-        return ZSTD_compressBlock_fast_generic(ms, seqStore, rep, src, srcSize, hlog, stepSize, 4, ZSTD_dictMatchState);
+        return ZSTD_compressBlock_fast_generic(ms, seqStore, rep, src, srcSize, 4, ZSTD_dictMatchState);
     case 5 :
-        return ZSTD_compressBlock_fast_generic(ms, seqStore, rep, src, srcSize, hlog, stepSize, 5, ZSTD_dictMatchState);
+        return ZSTD_compressBlock_fast_generic(ms, seqStore, rep, src, srcSize, 5, ZSTD_dictMatchState);
     case 6 :
-        return ZSTD_compressBlock_fast_generic(ms, seqStore, rep, src, srcSize, hlog, stepSize, 6, ZSTD_dictMatchState);
+        return ZSTD_compressBlock_fast_generic(ms, seqStore, rep, src, srcSize, 6, ZSTD_dictMatchState);
     case 7 :
-        return ZSTD_compressBlock_fast_generic(ms, seqStore, rep, src, srcSize, hlog, stepSize, 7, ZSTD_dictMatchState);
+        return ZSTD_compressBlock_fast_generic(ms, seqStore, rep, src, srcSize, 7, ZSTD_dictMatchState);
     }
 }
 
 
 static size_t ZSTD_compressBlock_fast_extDict_generic(
         ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
-        void const* src, size_t srcSize,
-        U32 const hlog, U32 stepSize, U32 const mls)
+        void const* src, size_t srcSize, U32 const mls)
 {
-    U32* hashTable = ms->hashTable;
+    const ZSTD_compressionParameters* const cParams = &ms->cParams;
+    U32* const hashTable = ms->hashTable;
+    U32 const hlog = cParams->hashLog;
+    /* support stepSize of 0 */
+    U32 const stepSize = cParams->targetLength + !(cParams->targetLength);
     const BYTE* const base = ms->window.base;
     const BYTE* const dictBase = ms->window.dictBase;
     const BYTE* const istart = (const BYTE*)src;
@@ -289,8 +290,6 @@ static size_t ZSTD_compressBlock_fast_extDict_generic(
     const BYTE* const ilimit = iend - 8;
     U32 offset_1=rep[0], offset_2=rep[1];
 
-    stepSize += !stepSize;   /* support stepSize == 0 */
-
     /* Search Loop */
     while (ip < ilimit) {  /* < instead of <=, because (ip+1) */
         const size_t h = ZSTD_hashPtr(ip, hlog, mls);
@@ -370,19 +369,17 @@ size_t ZSTD_compressBlock_fast_extDict(
         void const* src, size_t srcSize)
 {
     ZSTD_compressionParameters const* cParams = &ms->cParams;
-    U32 const hlog = cParams->hashLog;
     U32 const mls = cParams->searchLength;
-    U32 const stepSize = cParams->targetLength;
     switch(mls)
     {
     default: /* includes case 3 */
     case 4 :
-        return ZSTD_compressBlock_fast_extDict_generic(ms, seqStore, rep, src, srcSize, hlog, stepSize, 4);
+        return ZSTD_compressBlock_fast_extDict_generic(ms, seqStore, rep, src, srcSize, 4);
     case 5 :
-        return ZSTD_compressBlock_fast_extDict_generic(ms, seqStore, rep, src, srcSize, hlog, stepSize, 5);
+        return ZSTD_compressBlock_fast_extDict_generic(ms, seqStore, rep, src, srcSize, 5);
     case 6 :
-        return ZSTD_compressBlock_fast_extDict_generic(ms, seqStore, rep, src, srcSize, hlog, stepSize, 6);
+        return ZSTD_compressBlock_fast_extDict_generic(ms, seqStore, rep, src, srcSize, 6);
     case 7 :
-        return ZSTD_compressBlock_fast_extDict_generic(ms, seqStore, rep, src, srcSize, hlog, stepSize, 7);
+        return ZSTD_compressBlock_fast_extDict_generic(ms, seqStore, rep, src, srcSize, 7);
     }
 }

From fe96e98f816e66f757a84ef2930c4a72ee662ddb Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" 
Date: Thu, 23 Aug 2018 14:49:09 -0700
Subject: [PATCH 344/372] Support a Separate Hash Log in ZSTD_fast

---
 lib/compress/zstd_fast.c | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/lib/compress/zstd_fast.c b/lib/compress/zstd_fast.c
index bb7234846..dfb46abe4 100644
--- a/lib/compress/zstd_fast.c
+++ b/lib/compress/zstd_fast.c
@@ -64,6 +64,9 @@ size_t ZSTD_compressBlock_fast_generic(
     U32 offsetSaved = 0;
 
     const ZSTD_matchState_t* const dms = ms->dictMatchState;
+    const ZSTD_compressionParameters* const dictCParams =
+                                     dictMode == ZSTD_dictMatchState ?
+                                     &dms->cParams : NULL;
     const U32* const dictHashTable = dictMode == ZSTD_dictMatchState ?
                                      dms->hashTable : NULL;
     const U32 dictStartIndex       = dictMode == ZSTD_dictMatchState ?
@@ -78,6 +81,8 @@ size_t ZSTD_compressBlock_fast_generic(
                                      prefixStartIndex - (U32)(dictEnd - dictBase) :
                                      0;
     const U32 dictAndPrefixLength  = (U32)(ip - prefixStart + dictEnd - dictStart);
+    const U32 dictHLog             = dictMode == ZSTD_dictMatchState ?
+                                     dictCParams->hashLog : 0;
 
     assert(dictMode == ZSTD_noDict || dictMode == ZSTD_dictMatchState);
 
@@ -128,7 +133,8 @@ size_t ZSTD_compressBlock_fast_generic(
             ZSTD_storeSeq(seqStore, ip-anchor, anchor, 0, mLength-MINMATCH);
         } else if ( (matchIndex <= prefixStartIndex) ) {
             if (dictMode == ZSTD_dictMatchState) {
-                U32 const dictMatchIndex = dictHashTable[h];
+                size_t const dictHash = ZSTD_hashPtr(ip, dictHLog, mls);
+                U32 const dictMatchIndex = dictHashTable[dictHash];
                 const BYTE* dictMatch = dictBase + dictMatchIndex;
                 if (dictMatchIndex <= dictStartIndex ||
                     MEM_read32(dictMatch) != MEM_read32(ip)) {

From a232b3bb7c0acb80b4ec4e25d05d2146c52999e0 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" 
Date: Mon, 27 Aug 2018 15:23:27 -0700
Subject: [PATCH 345/372] Bump Split Log Support to ZSTD_dfast

---
 lib/compress/zstd_compress.c | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index 4d54772a0..8fa42bab8 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -1332,8 +1332,8 @@ static int ZSTD_shouldAttachDict(ZSTD_CCtx* cctx,
         && params.attachDictPref != ZSTD_dictForceCopy
         && !params.forceWindow /* dictMatchState isn't correctly
                                 * handled in _enforceMaxDist */
-        && ( (cdict->matchState.cParams.strategy <= ZSTD_fast)
-          || (cdict->matchState.cParams.strategy > ZSTD_fast &&
+        && ( (cdict->matchState.cParams.strategy <= ZSTD_dfast)
+          || (cdict->matchState.cParams.strategy > ZSTD_dfast &&
               ZSTD_equivalentCParams(cctx->appliedParams.cParams,
                                      cdict->matchState.cParams)));
 }
@@ -1351,7 +1351,7 @@ static size_t ZSTD_resetCCtx_byAttachingCDict(ZSTD_CCtx* cctx,
         /* Copy only compression parameters related to tables. */
         params.cParams = *cdict_cParams;
         params.cParams.windowLog = windowLog;
-        if (params.cParams.strategy <= ZSTD_fast) {
+        if (params.cParams.strategy <= ZSTD_dfast) {
             DEBUGLOG(4, "Overriding hashLog from %d to %d", params.cParams.hashLog, cctx->requestedParams.cParams.hashLog);
             DEBUGLOG(4, "Overriding chainLog from %d to %d", params.cParams.chainLog, cctx->requestedParams.cParams.chainLog);
             if (cctx->requestedParams.cParams.hashLog)
@@ -1362,7 +1362,7 @@ static size_t ZSTD_resetCCtx_byAttachingCDict(ZSTD_CCtx* cctx,
         ZSTD_resetCCtx_internal(cctx, params, pledgedSrcSize,
                                 ZSTDcrp_continue, zbuff);
         assert(cctx->appliedParams.cParams.strategy == cdict_cParams->strategy);
-        if (params.cParams.strategy > ZSTD_fast) {
+        if (params.cParams.strategy > ZSTD_dfast) {
             assert(cctx->appliedParams.cParams.hashLog == cdict_cParams->hashLog);
             assert(cctx->appliedParams.cParams.chainLog == cdict_cParams->chainLog);
         }

From 22fcb8d4c71d8482bf64090029e14cb8078d1cf5 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" 
Date: Mon, 27 Aug 2018 15:23:51 -0700
Subject: [PATCH 346/372] Support Split Logs in ZSTD_dfast

---
 lib/compress/zstd_double_fast.c | 16 +++++++++++++---
 1 file changed, 13 insertions(+), 3 deletions(-)

diff --git a/lib/compress/zstd_double_fast.c b/lib/compress/zstd_double_fast.c
index e27104206..9a2cf8983 100644
--- a/lib/compress/zstd_double_fast.c
+++ b/lib/compress/zstd_double_fast.c
@@ -71,6 +71,9 @@ size_t ZSTD_compressBlock_doubleFast_generic(
     U32 offsetSaved = 0;
 
     const ZSTD_matchState_t* const dms = ms->dictMatchState;
+    const ZSTD_compressionParameters* const dictCParams =
+                                     dictMode == ZSTD_dictMatchState ?
+                                     &dms->cParams : NULL;
     const U32* const dictHashLong  = dictMode == ZSTD_dictMatchState ?
                                      dms->hashTable : NULL;
     const U32* const dictHashSmall = dictMode == ZSTD_dictMatchState ?
@@ -86,6 +89,10 @@ size_t ZSTD_compressBlock_doubleFast_generic(
     const U32 dictIndexDelta       = dictMode == ZSTD_dictMatchState ?
                                      prefixLowestIndex - (U32)(dictEnd - dictBase) :
                                      0;
+    const U32 dictHBitsL           = dictMode == ZSTD_dictMatchState ?
+                                     dictCParams->hashLog : 0;
+    const U32 dictHBitsS           = dictMode == ZSTD_dictMatchState ?
+                                     dictCParams->chainLog : 0;
     const U32 dictAndPrefixLength  = (U32)(ip - prefixLowest + dictEnd - dictStart);
 
     assert(dictMode == ZSTD_noDict || dictMode == ZSTD_dictMatchState);
@@ -110,6 +117,8 @@ size_t ZSTD_compressBlock_doubleFast_generic(
         U32 offset;
         size_t const h2 = ZSTD_hashPtr(ip, hBitsL, 8);
         size_t const h = ZSTD_hashPtr(ip, hBitsS, mls);
+        size_t const dictHL = ZSTD_hashPtr(ip, dictHBitsL, 8);
+        size_t const dictHS = ZSTD_hashPtr(ip, dictHBitsS, mls);
         U32 const current = (U32)(ip-base);
         U32 const matchIndexL = hashLong[h2];
         U32 matchIndexS = hashSmall[h];
@@ -152,7 +161,7 @@ size_t ZSTD_compressBlock_doubleFast_generic(
             }
         } else if (dictMode == ZSTD_dictMatchState) {
             /* check dictMatchState long match */
-            U32 const dictMatchIndexL = dictHashLong[h2];
+            U32 const dictMatchIndexL = dictHashLong[dictHL];
             const BYTE* dictMatchL = dictBase + dictMatchIndexL;
             assert(dictMatchL < dictEnd);
 
@@ -171,7 +180,7 @@ size_t ZSTD_compressBlock_doubleFast_generic(
             }
         } else if (dictMode == ZSTD_dictMatchState) {
             /* check dictMatchState short match */
-            U32 const dictMatchIndexS = dictHashSmall[h];
+            U32 const dictMatchIndexS = dictHashSmall[dictHS];
             match = dictBase + dictMatchIndexS;
             matchIndexS = dictMatchIndexS + dictIndexDelta;
 
@@ -187,6 +196,7 @@ _search_next_long:
 
         {
             size_t const hl3 = ZSTD_hashPtr(ip+1, hBitsL, 8);
+            size_t const dictHLNext = ZSTD_hashPtr(ip+1, dictHBitsL, 8);
             U32 const matchIndexL3 = hashLong[hl3];
             const BYTE* matchL3 = base + matchIndexL3;
             hashLong[hl3] = current + 1;
@@ -202,7 +212,7 @@ _search_next_long:
                 }
             } else if (dictMode == ZSTD_dictMatchState) {
                 /* check dict long +1 match */
-                U32 const dictMatchIndexL3 = dictHashLong[hl3];
+                U32 const dictMatchIndexL3 = dictHashLong[dictHLNext];
                 const BYTE* dictMatchL3 = dictBase + dictMatchIndexL3;
                 assert(dictMatchL3 < dictEnd);
                 if (dictMatchL3 > dictStart && MEM_read64(dictMatchL3) == MEM_read64(ip+1)) {

From e710dc336955918a2121671c0e6579b7b7ce875f Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" 
Date: Mon, 27 Aug 2018 16:19:24 -0700
Subject: [PATCH 347/372] Bump Split Log Support to ZSTD_btlazy2

---
 lib/compress/zstd_compress.c | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index 8fa42bab8..ab4575566 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -1320,6 +1320,8 @@ static const size_t attachDictSizeCutoffs[(unsigned)ZSTD_btultra+1] = {
     8 KB /* ZSTD_btultra */
 };
 
+static const ZSTD_strategy splitLogCutoffStrategy = ZSTD_btlazy2;
+
 static int ZSTD_shouldAttachDict(ZSTD_CCtx* cctx,
                             const ZSTD_CDict* cdict,
                             ZSTD_CCtx_params params,
@@ -1332,8 +1334,8 @@ static int ZSTD_shouldAttachDict(ZSTD_CCtx* cctx,
         && params.attachDictPref != ZSTD_dictForceCopy
         && !params.forceWindow /* dictMatchState isn't correctly
                                 * handled in _enforceMaxDist */
-        && ( (cdict->matchState.cParams.strategy <= ZSTD_dfast)
-          || (cdict->matchState.cParams.strategy > ZSTD_dfast &&
+        && ( (cdict->matchState.cParams.strategy <= splitLogCutoffStrategy)
+          || (cdict->matchState.cParams.strategy > splitLogCutoffStrategy &&
               ZSTD_equivalentCParams(cctx->appliedParams.cParams,
                                      cdict->matchState.cParams)));
 }
@@ -1351,7 +1353,7 @@ static size_t ZSTD_resetCCtx_byAttachingCDict(ZSTD_CCtx* cctx,
         /* Copy only compression parameters related to tables. */
         params.cParams = *cdict_cParams;
         params.cParams.windowLog = windowLog;
-        if (params.cParams.strategy <= ZSTD_dfast) {
+        if (params.cParams.strategy <= splitLogCutoffStrategy) {
             DEBUGLOG(4, "Overriding hashLog from %d to %d", params.cParams.hashLog, cctx->requestedParams.cParams.hashLog);
             DEBUGLOG(4, "Overriding chainLog from %d to %d", params.cParams.chainLog, cctx->requestedParams.cParams.chainLog);
             if (cctx->requestedParams.cParams.hashLog)
@@ -1362,7 +1364,7 @@ static size_t ZSTD_resetCCtx_byAttachingCDict(ZSTD_CCtx* cctx,
         ZSTD_resetCCtx_internal(cctx, params, pledgedSrcSize,
                                 ZSTDcrp_continue, zbuff);
         assert(cctx->appliedParams.cParams.strategy == cdict_cParams->strategy);
-        if (params.cParams.strategy > ZSTD_dfast) {
+        if (params.cParams.strategy > splitLogCutoffStrategy) {
             assert(cctx->appliedParams.cParams.hashLog == cdict_cParams->hashLog);
             assert(cctx->appliedParams.cParams.chainLog == cdict_cParams->chainLog);
         }

From e4ac4a0f168961568598fe2ee6eec27d9229a929 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" 
Date: Mon, 27 Aug 2018 16:19:41 -0700
Subject: [PATCH 348/372] Support Split Logs in ZSTD_greedy..ZSTD_btlazy2

---
 lib/compress/zstd_lazy.c | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/lib/compress/zstd_lazy.c b/lib/compress/zstd_lazy.c
index 725623d22..98d95954e 100644
--- a/lib/compress/zstd_lazy.c
+++ b/lib/compress/zstd_lazy.c
@@ -150,10 +150,10 @@ static size_t ZSTD_DUBT_findBetterDictMatch (
         U32 nbCompares,
         U32 const mls,
         const ZSTD_dictMode_e dictMode) {
-    const ZSTD_compressionParameters* const cParams = &ms->cParams;
     const ZSTD_matchState_t * const dms = ms->dictMatchState;
+    const ZSTD_compressionParameters* const dmsCParams = &dms->cParams;
     const U32 * const dictHashTable = dms->hashTable;
-    U32         const hashLog = cParams->hashLog;
+    U32         const hashLog = dmsCParams->hashLog;
     size_t      const h  = ZSTD_hashPtr(ip, hashLog, mls);
     U32               dictMatchIndex = dictHashTable[h];
 
@@ -167,7 +167,7 @@ static size_t ZSTD_DUBT_findBetterDictMatch (
     U32         const dictIndexDelta = ms->window.lowLimit - dictHighLimit;
 
     U32*        const dictBt = dms->chainTable;
-    U32         const btLog  = cParams->chainLog - 1;
+    U32         const btLog  = dmsCParams->chainLog - 1;
     U32         const btMask = (1 << btLog) - 1;
     U32         const btLow = (btMask >= dictHighLimit - dictLowLimit) ? dictLowLimit : dictHighLimit - btMask;
 
@@ -512,14 +512,16 @@ size_t ZSTD_HcFindBestMatch_generic (
     if (dictMode == ZSTD_dictMatchState) {
         const ZSTD_matchState_t* const dms = ms->dictMatchState;
         const U32* const dmsChainTable = dms->chainTable;
+        const U32 dmsChainSize         = (1 << dms->cParams.chainLog);
+        const U32 dmsChainMask         = dmsChainSize - 1;
         const U32 dmsLowestIndex       = dms->window.dictLimit;
         const BYTE* const dmsBase      = dms->window.base;
         const BYTE* const dmsEnd       = dms->window.nextSrc;
         const U32 dmsSize              = (U32)(dmsEnd - dmsBase);
         const U32 dmsIndexDelta        = dictLimit - dmsSize;
-        const U32 dmsMinChain = dmsSize > chainSize ? dmsSize - chainSize : 0;
+        const U32 dmsMinChain = dmsSize > dmsChainSize ? dmsSize - dmsChainSize : 0;
 
-        matchIndex = dms->hashTable[ZSTD_hashPtr(ip, cParams->hashLog, mls)];
+        matchIndex = dms->hashTable[ZSTD_hashPtr(ip, dms->cParams.hashLog, mls)];
 
         for ( ; (matchIndex>dmsLowestIndex) & (nbAttempts>0) ; nbAttempts--) {
             size_t currentMl=0;
@@ -536,7 +538,7 @@ size_t ZSTD_HcFindBestMatch_generic (
             }
 
             if (matchIndex <= dmsMinChain) break;
-            matchIndex = dmsChainTable[matchIndex & chainMask];
+            matchIndex = dmsChainTable[matchIndex & dmsChainMask];
         }
     }
 

From 0783492178292066b6232c2411a6976ab78f1766 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" 
Date: Mon, 27 Aug 2018 16:36:52 -0700
Subject: [PATCH 349/372] Bump Split Log Support to ZSTD_btultra

---
 lib/compress/zstd_compress.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index ab4575566..7ec9c176c 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -1320,7 +1320,7 @@ static const size_t attachDictSizeCutoffs[(unsigned)ZSTD_btultra+1] = {
     8 KB /* ZSTD_btultra */
 };
 
-static const ZSTD_strategy splitLogCutoffStrategy = ZSTD_btlazy2;
+static const ZSTD_strategy splitLogCutoffStrategy = ZSTD_btultra;
 
 static int ZSTD_shouldAttachDict(ZSTD_CCtx* cctx,
                             const ZSTD_CDict* cdict,

From 00c088b32d7e8392e917025b5b859f92035124e6 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" 
Date: Mon, 27 Aug 2018 16:37:04 -0700
Subject: [PATCH 350/372] Support Split Logs in ZSTD_btopt..ZSTD_btultra

---
 lib/compress/zstd_opt.c | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/lib/compress/zstd_opt.c b/lib/compress/zstd_opt.c
index 25d3487a7..170265649 100644
--- a/lib/compress/zstd_opt.c
+++ b/lib/compress/zstd_opt.c
@@ -526,12 +526,17 @@ U32 ZSTD_insertBtAndGetAllMatches (
     U32 nbCompares = 1U << cParams->searchLog;
 
     const ZSTD_matchState_t* dms    = dictMode == ZSTD_dictMatchState ? ms->dictMatchState : NULL;
+    const ZSTD_compressionParameters* const dmsCParams =
+                                      dictMode == ZSTD_dictMatchState ? &dms->cParams : NULL;
     const BYTE* const dmsBase       = dictMode == ZSTD_dictMatchState ? dms->window.base : NULL;
     const BYTE* const dmsEnd        = dictMode == ZSTD_dictMatchState ? dms->window.nextSrc : NULL;
     U32         const dmsHighLimit  = dictMode == ZSTD_dictMatchState ? (U32)(dmsEnd - dmsBase) : 0;
     U32         const dmsLowLimit   = dictMode == ZSTD_dictMatchState ? dms->window.lowLimit : 0;
     U32         const dmsIndexDelta = dictMode == ZSTD_dictMatchState ? windowLow - dmsHighLimit : 0;
-    U32         const dmsBtLow      = dictMode == ZSTD_dictMatchState && btMask < dmsHighLimit - dmsLowLimit ? dmsHighLimit - btMask : dmsLowLimit;
+    U32         const dmsHashLog    = dictMode == ZSTD_dictMatchState ? dmsCParams->hashLog : 0;
+    U32         const dmsBtLog      = dictMode == ZSTD_dictMatchState ? dmsCParams->chainLog - 1 : 0;
+    U32         const dmsBtMask     = dictMode == ZSTD_dictMatchState ? (1U << dmsBtLog) - 1 : 0;
+    U32         const dmsBtLow      = dictMode == ZSTD_dictMatchState && dmsBtMask < dmsHighLimit - dmsLowLimit ? dmsHighLimit - dmsBtMask : dmsLowLimit;
 
     size_t bestLength = lengthToBeat-1;
     DEBUGLOG(8, "ZSTD_insertBtAndGetAllMatches: current=%u", current);
@@ -666,11 +671,12 @@ U32 ZSTD_insertBtAndGetAllMatches (
     *smallerPtr = *largerPtr = 0;
 
     if (dictMode == ZSTD_dictMatchState && nbCompares) {
-        U32 dictMatchIndex = dms->hashTable[h];
+        size_t const dmsH = ZSTD_hashPtr(ip, dmsHashLog, mls);
+        U32 dictMatchIndex = dms->hashTable[dmsH];
         const U32* const dmsBt = dms->chainTable;
         commonLengthSmaller = commonLengthLarger = 0;
         while (nbCompares-- && (dictMatchIndex > dmsLowLimit)) {
-            const U32* const nextPtr = dmsBt + 2*(dictMatchIndex & btMask);
+            const U32* const nextPtr = dmsBt + 2*(dictMatchIndex & dmsBtMask);
             size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger);   /* guaranteed minimum nb of common bytes */
             const BYTE* match = dmsBase + dictMatchIndex;
             matchLength += ZSTD_count_2segments(ip+matchLength, match+matchLength, iLimit, dmsEnd, prefixStart);

From 77fd17d93f73c047c0023082c87ac55303f0d7e1 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" 
Date: Wed, 12 Sep 2018 12:04:04 -0700
Subject: [PATCH 351/372] Remove Strategy-Dependency in Making Attachment
 Decision

---
 lib/compress/zstd_compress.c | 48 ++++++++++++++----------------------
 1 file changed, 18 insertions(+), 30 deletions(-)

diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index 7ec9c176c..b823eb7c4 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -1320,31 +1320,25 @@ static const size_t attachDictSizeCutoffs[(unsigned)ZSTD_btultra+1] = {
     8 KB /* ZSTD_btultra */
 };
 
-static const ZSTD_strategy splitLogCutoffStrategy = ZSTD_btultra;
-
-static int ZSTD_shouldAttachDict(ZSTD_CCtx* cctx,
-                            const ZSTD_CDict* cdict,
-                            ZSTD_CCtx_params params,
-                            U64 pledgedSrcSize)
+static int ZSTD_shouldAttachDict(const ZSTD_CDict* cdict,
+                                 ZSTD_CCtx_params params,
+                                 U64 pledgedSrcSize)
 {
     size_t cutoff = attachDictSizeCutoffs[cdict->matchState.cParams.strategy];
     return ( pledgedSrcSize <= cutoff
           || pledgedSrcSize == ZSTD_CONTENTSIZE_UNKNOWN
           || params.attachDictPref == ZSTD_dictForceAttach )
         && params.attachDictPref != ZSTD_dictForceCopy
-        && !params.forceWindow /* dictMatchState isn't correctly
-                                * handled in _enforceMaxDist */
-        && ( (cdict->matchState.cParams.strategy <= splitLogCutoffStrategy)
-          || (cdict->matchState.cParams.strategy > splitLogCutoffStrategy &&
-              ZSTD_equivalentCParams(cctx->appliedParams.cParams,
-                                     cdict->matchState.cParams)));
+        && !params.forceWindow; /* dictMatchState isn't correctly
+                                 * handled in _enforceMaxDist */
 }
 
-static size_t ZSTD_resetCCtx_byAttachingCDict(ZSTD_CCtx* cctx,
-                            const ZSTD_CDict* cdict,
-                            ZSTD_CCtx_params params,
-                            U64 pledgedSrcSize,
-                            ZSTD_buffered_policy_e zbuff)
+static size_t ZSTD_resetCCtx_byAttachingCDict(
+    ZSTD_CCtx* cctx,
+    const ZSTD_CDict* cdict,
+    ZSTD_CCtx_params params,
+    U64 pledgedSrcSize,
+    ZSTD_buffered_policy_e zbuff)
 {
     {
         const ZSTD_compressionParameters *cdict_cParams = &cdict->matchState.cParams;
@@ -1353,21 +1347,15 @@ static size_t ZSTD_resetCCtx_byAttachingCDict(ZSTD_CCtx* cctx,
         /* Copy only compression parameters related to tables. */
         params.cParams = *cdict_cParams;
         params.cParams.windowLog = windowLog;
-        if (params.cParams.strategy <= splitLogCutoffStrategy) {
-            DEBUGLOG(4, "Overriding hashLog from %d to %d", params.cParams.hashLog, cctx->requestedParams.cParams.hashLog);
-            DEBUGLOG(4, "Overriding chainLog from %d to %d", params.cParams.chainLog, cctx->requestedParams.cParams.chainLog);
-            if (cctx->requestedParams.cParams.hashLog)
-                params.cParams.hashLog = cctx->requestedParams.cParams.hashLog;
-            if (cctx->requestedParams.cParams.chainLog)
-                params.cParams.chainLog = cctx->requestedParams.cParams.chainLog;
-        }
+        DEBUGLOG(4, "Overriding hashLog from %d to %d", params.cParams.hashLog, cctx->requestedParams.cParams.hashLog);
+        DEBUGLOG(4, "Overriding chainLog from %d to %d", params.cParams.chainLog, cctx->requestedParams.cParams.chainLog);
+        if (cctx->requestedParams.cParams.hashLog)
+            params.cParams.hashLog = cctx->requestedParams.cParams.hashLog;
+        if (cctx->requestedParams.cParams.chainLog)
+            params.cParams.chainLog = cctx->requestedParams.cParams.chainLog;
         ZSTD_resetCCtx_internal(cctx, params, pledgedSrcSize,
                                 ZSTDcrp_continue, zbuff);
         assert(cctx->appliedParams.cParams.strategy == cdict_cParams->strategy);
-        if (params.cParams.strategy > splitLogCutoffStrategy) {
-            assert(cctx->appliedParams.cParams.hashLog == cdict_cParams->hashLog);
-            assert(cctx->appliedParams.cParams.chainLog == cdict_cParams->chainLog);
-        }
     }
 
     {
@@ -1468,7 +1456,7 @@ static size_t ZSTD_resetCCtx_usingCDict(ZSTD_CCtx* cctx,
 
     DEBUGLOG(4, "ZSTD_resetCCtx_usingCDict (pledgedSrcSize=%u)", (U32)pledgedSrcSize);
 
-    if (ZSTD_shouldAttachDict(cctx, cdict, params, pledgedSrcSize)) {
+    if (ZSTD_shouldAttachDict(cdict, params, pledgedSrcSize)) {
         return ZSTD_resetCCtx_byAttachingCDict(
             cctx, cdict, params, pledgedSrcSize, zbuff);
     } else {

From 9d87d50878c80db732441bd15feecc6dd70d4354 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" 
Date: Wed, 12 Sep 2018 12:28:53 -0700
Subject: [PATCH 352/372] Remove Log Overriding for the Time Being

---
 lib/compress/zstd_compress.c | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index b823eb7c4..da8bd1e1e 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -1347,12 +1347,6 @@ static size_t ZSTD_resetCCtx_byAttachingCDict(
         /* Copy only compression parameters related to tables. */
         params.cParams = *cdict_cParams;
         params.cParams.windowLog = windowLog;
-        DEBUGLOG(4, "Overriding hashLog from %d to %d", params.cParams.hashLog, cctx->requestedParams.cParams.hashLog);
-        DEBUGLOG(4, "Overriding chainLog from %d to %d", params.cParams.chainLog, cctx->requestedParams.cParams.chainLog);
-        if (cctx->requestedParams.cParams.hashLog)
-            params.cParams.hashLog = cctx->requestedParams.cParams.hashLog;
-        if (cctx->requestedParams.cParams.chainLog)
-            params.cParams.chainLog = cctx->requestedParams.cParams.chainLog;
         ZSTD_resetCCtx_internal(cctx, params, pledgedSrcSize,
                                 ZSTDcrp_continue, zbuff);
         assert(cctx->appliedParams.cParams.strategy == cdict_cParams->strategy);

From c38acff94f296114f84464387f94f51b21b22d3a Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" 
Date: Wed, 12 Sep 2018 12:29:53 -0700
Subject: [PATCH 353/372] When Attaching Dictionary, Size Working Tables Based
 on Input Size Only

---
 lib/compress/zstd_compress.c | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index da8bd1e1e..2bd672c15 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -1344,8 +1344,9 @@ static size_t ZSTD_resetCCtx_byAttachingCDict(
         const ZSTD_compressionParameters *cdict_cParams = &cdict->matchState.cParams;
         unsigned const windowLog = params.cParams.windowLog;
         assert(windowLog != 0);
-        /* Copy only compression parameters related to tables. */
-        params.cParams = *cdict_cParams;
+        /* Resize working context table params for input only, since the dict
+         * has its own tables. */
+        params.cParams = ZSTD_adjustCParams_internal(*cdict_cParams, pledgedSrcSize, 0);
         params.cParams.windowLog = windowLog;
         ZSTD_resetCCtx_internal(cctx, params, pledgedSrcSize,
                                 ZSTDcrp_continue, zbuff);

From bad74c47819fe8aa635b9f10abafb80c6987697d Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" 
Date: Thu, 13 Sep 2018 11:13:08 -0700
Subject: [PATCH 354/372] Use Working Ctx Logs when not in DMS Mode

We pre-hash the ptr for the dict match state sometimes. When that actually
happens, a hashlog of 0 can produce undefined behavior (right shift a long
long by 64). Only applies to unoptimized compilations, since when
optimizations are applied, those hash operations are dropped when we're not
actually in dms mode.
---
 lib/compress/zstd_double_fast.c | 4 ++--
 lib/compress/zstd_fast.c        | 2 +-
 lib/compress/zstd_opt.c         | 4 ++--
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/lib/compress/zstd_double_fast.c b/lib/compress/zstd_double_fast.c
index 9a2cf8983..7b9e18e7e 100644
--- a/lib/compress/zstd_double_fast.c
+++ b/lib/compress/zstd_double_fast.c
@@ -90,9 +90,9 @@ size_t ZSTD_compressBlock_doubleFast_generic(
                                      prefixLowestIndex - (U32)(dictEnd - dictBase) :
                                      0;
     const U32 dictHBitsL           = dictMode == ZSTD_dictMatchState ?
-                                     dictCParams->hashLog : 0;
+                                     dictCParams->hashLog : hBitsL;
     const U32 dictHBitsS           = dictMode == ZSTD_dictMatchState ?
-                                     dictCParams->chainLog : 0;
+                                     dictCParams->chainLog : hBitsS;
     const U32 dictAndPrefixLength  = (U32)(ip - prefixLowest + dictEnd - dictStart);
 
     assert(dictMode == ZSTD_noDict || dictMode == ZSTD_dictMatchState);
diff --git a/lib/compress/zstd_fast.c b/lib/compress/zstd_fast.c
index dfb46abe4..247746517 100644
--- a/lib/compress/zstd_fast.c
+++ b/lib/compress/zstd_fast.c
@@ -82,7 +82,7 @@ size_t ZSTD_compressBlock_fast_generic(
                                      0;
     const U32 dictAndPrefixLength  = (U32)(ip - prefixStart + dictEnd - dictStart);
     const U32 dictHLog             = dictMode == ZSTD_dictMatchState ?
-                                     dictCParams->hashLog : 0;
+                                     dictCParams->hashLog : hlog;
 
     assert(dictMode == ZSTD_noDict || dictMode == ZSTD_dictMatchState);
 
diff --git a/lib/compress/zstd_opt.c b/lib/compress/zstd_opt.c
index 170265649..8af69a91d 100644
--- a/lib/compress/zstd_opt.c
+++ b/lib/compress/zstd_opt.c
@@ -533,8 +533,8 @@ U32 ZSTD_insertBtAndGetAllMatches (
     U32         const dmsHighLimit  = dictMode == ZSTD_dictMatchState ? (U32)(dmsEnd - dmsBase) : 0;
     U32         const dmsLowLimit   = dictMode == ZSTD_dictMatchState ? dms->window.lowLimit : 0;
     U32         const dmsIndexDelta = dictMode == ZSTD_dictMatchState ? windowLow - dmsHighLimit : 0;
-    U32         const dmsHashLog    = dictMode == ZSTD_dictMatchState ? dmsCParams->hashLog : 0;
-    U32         const dmsBtLog      = dictMode == ZSTD_dictMatchState ? dmsCParams->chainLog - 1 : 0;
+    U32         const dmsHashLog    = dictMode == ZSTD_dictMatchState ? dmsCParams->hashLog : hashLog;
+    U32         const dmsBtLog      = dictMode == ZSTD_dictMatchState ? dmsCParams->chainLog - 1 : btLog;
     U32         const dmsBtMask     = dictMode == ZSTD_dictMatchState ? (1U << dmsBtLog) - 1 : 0;
     U32         const dmsBtLow      = dictMode == ZSTD_dictMatchState && dmsBtMask < dmsHighLimit - dmsLowLimit ? dmsHighLimit - dmsBtMask : dmsLowLimit;
 

From c2369fedc4233c85c5f2a2a7de61f87593f9cb1a Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" 
Date: Wed, 26 Sep 2018 10:56:28 -0700
Subject: [PATCH 355/372] Restore Passing CParams to
 `ZSTD_insertAndFindFirstIndex_internal`

---
 lib/compress/zstd_lazy.c | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/lib/compress/zstd_lazy.c b/lib/compress/zstd_lazy.c
index 98d95954e..fb9ee3455 100644
--- a/lib/compress/zstd_lazy.c
+++ b/lib/compress/zstd_lazy.c
@@ -432,9 +432,9 @@ static size_t ZSTD_BtFindBestMatch_extDict_selectMLS (
    Assumption : always within prefix (i.e. not within extDict) */
 static U32 ZSTD_insertAndFindFirstIndex_internal(
                         ZSTD_matchState_t* ms,
+                        const ZSTD_compressionParameters* const cParams,
                         const BYTE* ip, U32 const mls)
 {
-    const ZSTD_compressionParameters* const cParams = &ms->cParams;
     U32* const hashTable  = ms->hashTable;
     const U32 hashLog = cParams->hashLog;
     U32* const chainTable = ms->chainTable;
@@ -455,7 +455,8 @@ static U32 ZSTD_insertAndFindFirstIndex_internal(
 }
 
 U32 ZSTD_insertAndFindFirstIndex(ZSTD_matchState_t* ms, const BYTE* ip) {
-    return ZSTD_insertAndFindFirstIndex_internal(ms, ip, ms->cParams.searchLength);
+    const ZSTD_compressionParameters* const cParams = &ms->cParams;
+    return ZSTD_insertAndFindFirstIndex_internal(ms, cParams, ip, ms->cParams.searchLength);
 }
 
 
@@ -483,7 +484,7 @@ size_t ZSTD_HcFindBestMatch_generic (
     size_t ml=4-1;
 
     /* HC4 match finder */
-    U32 matchIndex = ZSTD_insertAndFindFirstIndex_internal(ms, ip, mls);
+    U32 matchIndex = ZSTD_insertAndFindFirstIndex_internal(ms, cParams, ip, mls);
 
     for ( ; (matchIndex>lowLimit) & (nbAttempts>0) ; nbAttempts--) {
         size_t currentMl=0;

From 1ab71a8e7254ff8ab84cbf5b11698c0fcaf85a85 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Fri, 28 Sep 2018 18:19:23 -0700
Subject: [PATCH 356/372] regroup name creation logic into its own function

for a cleaner main file decompression loop
---
 programs/fileio.c | 126 ++++++++++++++++++++++++++--------------------
 1 file changed, 71 insertions(+), 55 deletions(-)

diff --git a/programs/fileio.c b/programs/fileio.c
index 12d24603f..e1903ce5e 100644
--- a/programs/fileio.c
+++ b/programs/fileio.c
@@ -1895,7 +1895,74 @@ int FIO_decompressFilename(const char* dstFileName, const char* srcFileName,
 }
 
 
-#define MAXSUFFIXSIZE 8
+/* FIO_determineDstName() :
+ * create a destination filename from a srcFileName.
+ * @return a pointer to it.
+ * @return == NULL if there is an error */
+static const char*
+FIO_determineDstName(const char* srcFileName)
+{
+    static size_t dfnbCapacity = 0;
+    static char* dstFileNameBuffer = NULL;   /* using static allocation : this function cannot be multi-threaded */
+
+    size_t const sfnSize = strlen(srcFileName);
+    size_t suffixSize;
+    const char* const suffixPtr = strrchr(srcFileName, '.');
+    if (suffixPtr == NULL) {
+        DISPLAYLEVEL(1, "zstd: %s: unknown suffix -- ignored \n",
+                        srcFileName);
+        return NULL;
+    }
+    suffixSize = strlen(suffixPtr);
+
+    /* check suffix is authorized */
+    if (sfnSize <= suffixSize
+        || (   strcmp(suffixPtr, ZSTD_EXTENSION)
+        #ifdef ZSTD_GZDECOMPRESS
+            && strcmp(suffixPtr, GZ_EXTENSION)
+        #endif
+        #ifdef ZSTD_LZMADECOMPRESS
+            && strcmp(suffixPtr, XZ_EXTENSION)
+            && strcmp(suffixPtr, LZMA_EXTENSION)
+        #endif
+        #ifdef ZSTD_LZ4DECOMPRESS
+            && strcmp(suffixPtr, LZ4_EXTENSION)
+        #endif
+            ) ) {
+        const char* suffixlist = ZSTD_EXTENSION
+        #ifdef ZSTD_GZDECOMPRESS
+            "/" GZ_EXTENSION
+        #endif
+        #ifdef ZSTD_LZMADECOMPRESS
+            "/" XZ_EXTENSION "/" LZMA_EXTENSION
+        #endif
+        #ifdef ZSTD_LZ4DECOMPRESS
+            "/" LZ4_EXTENSION
+        #endif
+        ;
+        DISPLAYLEVEL(1, "zstd: %s: unknown suffix (%s expected) -- ignored \n",
+                     srcFileName, suffixlist);
+        return NULL;
+    }
+
+    /* allocate enough space to write dstFilename into it */
+    if (dfnbCapacity+suffixSize <= sfnSize+1) {
+        free(dstFileNameBuffer);
+        dfnbCapacity = sfnSize + 20;
+        dstFileNameBuffer = (char*)malloc(dfnbCapacity);
+        if (dstFileNameBuffer==NULL)
+            EXM_THROW(74, "not enough memory for dstFileName");
+    }
+
+    /* return dst name == src name truncated from suffix */
+    memcpy(dstFileNameBuffer, srcFileName, sfnSize - suffixSize);
+    dstFileNameBuffer[sfnSize-suffixSize] = '\0';
+    return dstFileNameBuffer;
+
+    /* note : dstFileNameBuffer memory is not going to be free */
+}
+
+
 int FIO_decompressMultipleFilenames(const char** srcNamesTable, unsigned nbFiles,
                                     const char* outFileName,
                                     const char* dictFileName)
@@ -1913,65 +1980,14 @@ int FIO_decompressMultipleFilenames(const char** srcNamesTable, unsigned nbFiles
         if (fclose(ress.dstFile))
             EXM_THROW(72, "Write error : cannot properly close output file");
     } else {
-        size_t suffixSize;
-        size_t dfnbCapacity = FNSPACE;
         unsigned u;
-        char* dstFileName = (char*)malloc(dfnbCapacity);
-        if (dstFileName==NULL)
-            EXM_THROW(73, "not enough memory for dstFileName");
         for (u=0; u
Date: Mon, 1 Oct 2018 13:28:13 -0700
Subject: [PATCH 357/372] Revert Ability to Set HashLog and ChainLog on Context
 When Dict is Attached

This capability is not needed / used in the current unit of work. I'll
re-introduce it later, when we start allowing users to override the deduced
working context logs.
---
 lib/compress/zstd_compress.c | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index 2bd672c15..6efffefba 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -284,11 +284,9 @@ size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, unsigned v
         if (cctx->cdict) return ERROR(stage_wrong);
         return ZSTD_CCtxParam_setParameter(&cctx->requestedParams, param, value);
 
+    case ZSTD_p_windowLog:
     case ZSTD_p_hashLog:
     case ZSTD_p_chainLog:
-        return ZSTD_CCtxParam_setParameter(&cctx->requestedParams, param, value);
-
-    case ZSTD_p_windowLog:
     case ZSTD_p_searchLog:
     case ZSTD_p_minMatch:
     case ZSTD_p_targetLength:

From c7bd6a41ab8ffef602884c8a9298bbba8faee0e2 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Mon, 1 Oct 2018 14:04:00 -0700
Subject: [PATCH 358/372] zstd -d -f do no longer erase destination file

when source file does not exist (#1082)
---
 programs/fileio.c  | 138 +++++++++++++++++++++++++--------------------
 tests/playTests.sh |   2 +-
 2 files changed, 77 insertions(+), 63 deletions(-)

diff --git a/programs/fileio.c b/programs/fileio.c
index e1903ce5e..a2c4ded1d 100644
--- a/programs/fileio.c
+++ b/programs/fileio.c
@@ -1792,11 +1792,72 @@ static int FIO_decompressFrames(dRess_t ress, FILE* srcFile,
     return 0;
 }
 
+/** FIO_decompressDstFile() :
+    open `dstFileName`,
+    or path-through if ress.dstFile is already != 0,
+    then start decompression process (FIO_decompressFrames()).
+    @return : 0 : OK
+              1 : operation aborted
+*/
+static int FIO_decompressDstFile(dRess_t ress, FILE* srcFile,
+                                 const char* dstFileName, const char* srcFileName)
+{
+    int result;
+    stat_t statbuf;
+    int transfer_permissions = 0;
+    int releaseDstFile = 0;
+
+    if (ress.dstFile == NULL) {
+        releaseDstFile = 1;
+
+        ress.dstFile = FIO_openDstFile(dstFileName);
+        if (ress.dstFile==0) return 1;
+
+        /* Must only be added after FIO_openDstFile() succeeds.
+         * Otherwise we may delete the destination file if it already exists,
+         * and the user presses Ctrl-C when asked if they wish to overwrite.
+         */
+        addHandler(dstFileName);
+
+        if ( strcmp(srcFileName, stdinmark)   /* special case : don't transfer permissions from stdin */
+          && UTIL_getFileStat(srcFileName, &statbuf) )
+            transfer_permissions = 1;
+    }
+
+
+    result = FIO_decompressFrames(ress, srcFile, dstFileName, srcFileName);
+
+    if (releaseDstFile) {
+        FILE* const dstFile = ress.dstFile;
+        clearHandler();
+        ress.dstFile = NULL;
+        if (fclose(dstFile)) {
+            DISPLAYLEVEL(1, "zstd: %s: %s \n", dstFileName, strerror(errno));
+            result = 1;
+        }
+
+        if ( (result != 0)  /* operation failure */
+          && strcmp(dstFileName, nulmark)     /* special case : don't remove() /dev/null (#316) */
+          && strcmp(dstFileName, stdoutmark)  /* special case : don't remove() stdout */
+          ) {
+            FIO_remove(dstFileName);  /* remove decompression artefact; note: don't do anything special if remove() fails */
+        } else {  /* operation success */
+            if ( strcmp(dstFileName, stdoutmark) /* special case : don't chmod stdout */
+              && strcmp(dstFileName, nulmark)    /* special case : don't chmod /dev/null */
+              && transfer_permissions )          /* file permissions correctly extracted from src */
+                UTIL_setFileStat(dstFileName, &statbuf);  /* transfer file permissions from src into dst */
+        }
+        signal(SIGINT, SIG_DFL);
+    }
+
+    return result;
+}
+
 
 /** FIO_decompressSrcFile() :
-    Decompression `srcFileName` into `ress.dstFile`
+    Open `srcFileName`, transfer control to decompressDstFile()
     @return : 0 : OK
-              1 : operation not started
+              1 : error
 */
 static int FIO_decompressSrcFile(dRess_t ress, const char* dstFileName, const char* srcFileName)
 {
@@ -1812,15 +1873,15 @@ static int FIO_decompressSrcFile(dRess_t ress, const char* dstFileName, const ch
     if (srcFile==NULL) return 1;
     ress.srcBufferLoaded = 0;
 
-    result = FIO_decompressFrames(ress, srcFile, dstFileName, srcFileName);
+    result = FIO_decompressDstFile(ress, srcFile, dstFileName, srcFileName);
 
     /* Close file */
     if (fclose(srcFile)) {
         DISPLAYLEVEL(1, "zstd: %s: %s \n", srcFileName, strerror(errno));  /* error should not happen */
         return 1;
     }
-    if ( g_removeSrcFile /* --rm */
-      && (result==0)     /* decompression successful */
+    if ( g_removeSrcFile  /* --rm */
+      && (result==0)      /* decompression successful */
       && strcmp(srcFileName, stdinmark) ) /* not stdin */ {
         /* We must clear the handler, since after this point calling it would
          * delete both the source and destination files.
@@ -1835,60 +1896,13 @@ static int FIO_decompressSrcFile(dRess_t ress, const char* dstFileName, const ch
 }
 
 
-/** FIO_decompressFile_extRess() :
-    decompress `srcFileName` into `dstFileName`
-    @return : 0 : OK
-              1 : operation aborted (src not available, dst already taken, etc.)
-*/
-static int FIO_decompressDstFile(dRess_t ress,
-                                 const char* dstFileName, const char* srcFileName)
-{
-    int result;
-    stat_t statbuf;
-    int stat_result = 0;
-
-    ress.dstFile = FIO_openDstFile(dstFileName);
-    if (ress.dstFile==0) return 1;
-    /* Must ony be added after FIO_openDstFile() succeeds.
-     * Otherwise we may delete the destination file if at already exists, and
-     * the user presses Ctrl-C when asked if they wish to overwrite.
-     */
-    addHandler(dstFileName);
-
-    if ( strcmp(srcFileName, stdinmark)
-      && UTIL_getFileStat(srcFileName, &statbuf) )
-        stat_result = 1;
-    result = FIO_decompressSrcFile(ress, dstFileName, srcFileName);
-    clearHandler();
-
-    if (fclose(ress.dstFile)) {
-        DISPLAYLEVEL(1, "zstd: %s: %s \n", dstFileName, strerror(errno));
-        result = 1;
-    }
-
-    if ( (result != 0)  /* operation failure */
-      && strcmp(dstFileName, nulmark)      /* special case : don't remove() /dev/null (#316) */
-      && strcmp(dstFileName, stdoutmark) ) /* special case : don't remove() stdout */
-        FIO_remove(dstFileName);  /* remove decompression artefact; note don't do anything special if remove() fails */
-    else {  /* operation success */
-        if ( strcmp(dstFileName, stdoutmark) /* special case : don't chmod stdout */
-          && strcmp(dstFileName, nulmark)    /* special case : don't chmod /dev/null */
-          && stat_result )                   /* file permissions correctly extracted from src */
-            UTIL_setFileStat(dstFileName, &statbuf);  /* transfer file permissions from src into dst */
-    }
-
-    signal(SIGINT, SIG_DFL);
-
-    return result;
-}
-
 
 int FIO_decompressFilename(const char* dstFileName, const char* srcFileName,
                            const char* dictFileName)
 {
     dRess_t const ress = FIO_createDResources(dictFileName);
 
-    int const decodingError = FIO_decompressDstFile(ress, dstFileName, srcFileName);
+    int const decodingError = FIO_decompressSrcFile(ress, dstFileName, srcFileName);
 
     FIO_freeDResources(ress);
     return decodingError;
@@ -1963,12 +1977,12 @@ FIO_determineDstName(const char* srcFileName)
 }
 
 
-int FIO_decompressMultipleFilenames(const char** srcNamesTable, unsigned nbFiles,
-                                    const char* outFileName,
-                                    const char* dictFileName)
+int
+FIO_decompressMultipleFilenames(const char* srcNamesTable[], unsigned nbFiles,
+                                const char* outFileName,
+                                const char* dictFileName)
 {
-    int skippedFiles = 0;
-    int missingFiles = 0;
+    int error = 0;
     dRess_t ress = FIO_createDResources(dictFileName);
 
     if (outFileName) {
@@ -1976,7 +1990,7 @@ int FIO_decompressMultipleFilenames(const char** srcNamesTable, unsigned nbFiles
         ress.dstFile = FIO_openDstFile(outFileName);
         if (ress.dstFile == 0) EXM_THROW(71, "cannot open %s", outFileName);
         for (u=0; u
Date: Mon, 1 Oct 2018 17:16:34 -0700
Subject: [PATCH 359/372] ./zstd -f do no longer overwrite destination file

if source file does not exist (#1082)
---
 programs/fileio.c  | 204 +++++++++++++++++++++++++++------------------
 programs/zstdcli.c |   2 +-
 tests/playTests.sh |  11 ++-
 3 files changed, 134 insertions(+), 83 deletions(-)

diff --git a/programs/fileio.c b/programs/fileio.c
index a2c4ded1d..b6acb3e16 100644
--- a/programs/fileio.c
+++ b/programs/fileio.c
@@ -1066,14 +1066,80 @@ FIO_compressFilename_internal(cRess_t ress,
 }
 
 
+/*! FIO_compressFilename_dstFile() :
+ *  open dstFileName, or pass-through if ress.dstFile != NULL,
+ *  then start compression with FIO_compressFilename_internal().
+ *  Manages source removal (--rm) and file permissions transfer.
+ *  note : ress.srcFile must be != NULL,
+ *  so reach this function through FIO_compressFilename_srcFile().
+ *  @return : 0 : compression completed correctly,
+ *            1 : pb
+ */
+static int FIO_compressFilename_dstFile(cRess_t ress,
+                                        const char* dstFileName,
+                                        const char* srcFileName,
+                                        int compressionLevel)
+{
+    int closeDstFile = 0;
+    int result;
+    stat_t statbuf;
+    int transfer_permissions = 0;
+
+    assert(ress.srcFile != NULL);
+
+    if (ress.dstFile == NULL) {
+        closeDstFile = 1;
+        DISPLAYLEVEL(6, "FIO_compressFilename_dstFile: opening dst: %s", dstFileName);
+        ress.dstFile = FIO_openDstFile(dstFileName);
+        if (ress.dstFile==NULL) return 1;  /* could not open dstFileName */
+        /* Must only be added after FIO_openDstFile() succeeds.
+         * Otherwise we may delete the destination file if it already exists,
+         * and the user presses Ctrl-C when asked if they wish to overwrite.
+         */
+        addHandler(dstFileName);
+
+        if ( strcmp (srcFileName, stdinmark)
+          && UTIL_getFileStat(srcFileName, &statbuf))
+            transfer_permissions = 1;
+    }
+
+    result = FIO_compressFilename_internal(ress, dstFileName, srcFileName, compressionLevel);
+
+    if (closeDstFile) {
+        FILE* const dstFile = ress.dstFile;
+        ress.dstFile = NULL;
+
+        clearHandler();
+
+        if (fclose(dstFile)) { /* error closing dstFile */
+            DISPLAYLEVEL(1, "zstd: %s: %s \n", dstFileName, strerror(errno));
+            result=1;
+        }
+        if ( (result != 0)  /* operation failure */
+          && strcmp(dstFileName, nulmark)     /* special case : don't remove() /dev/null */
+          && strcmp(dstFileName, stdoutmark)  /* special case : don't remove() stdout */
+          ) {
+            FIO_remove(dstFileName); /* remove compression artefact; note don't do anything special if remove() fails */
+        } else if ( strcmp(dstFileName, stdoutmark)
+                 && strcmp(dstFileName, nulmark)
+                 && transfer_permissions) {
+            UTIL_setFileStat(dstFileName, &statbuf);
+        }
+    }
+
+    return result;
+}
+
+
 /*! FIO_compressFilename_srcFile() :
- *  note : ress.destFile already opened
  *  @return : 0 : compression completed correctly,
  *            1 : missing or pb opening srcFileName
  */
-static int FIO_compressFilename_srcFile(cRess_t ress,
-                            const char* dstFileName, const char* srcFileName,
-                            int compressionLevel)
+static int
+FIO_compressFilename_srcFile(cRess_t ress,
+                             const char* dstFileName,
+                             const char* srcFileName,
+                             int compressionLevel)
 {
     int result;
 
@@ -1084,12 +1150,16 @@ static int FIO_compressFilename_srcFile(cRess_t ress,
     }
 
     ress.srcFile = FIO_openSrcFile(srcFileName);
-    if (!ress.srcFile) return 1;   /* srcFile could not be opened */
+    if (ress.srcFile == NULL) return 1;   /* srcFile could not be opened */
 
-    result = FIO_compressFilename_internal(ress, dstFileName, srcFileName, compressionLevel);
+    result = FIO_compressFilename_dstFile(ress, dstFileName, srcFileName, compressionLevel);
 
     fclose(ress.srcFile);
-    if (g_removeSrcFile /* --rm */ && !result && strcmp(srcFileName, stdinmark)) {
+    ress.srcFile = NULL;
+    if ( g_removeSrcFile   /* --rm */
+      && result == 0       /* success */
+      && strcmp(srcFileName, stdinmark)   /* exception : don't erase stdin */
+      ) {
         /* We must clear the handler, since after this point calling it would
          * delete both the source and destination files.
          */
@@ -1101,50 +1171,6 @@ static int FIO_compressFilename_srcFile(cRess_t ress,
 }
 
 
-/*! FIO_compressFilename_dstFile() :
- *  @return : 0 : compression completed correctly,
- *            1 : pb
- */
-static int FIO_compressFilename_dstFile(cRess_t ress,
-                                        const char* dstFileName,
-                                        const char* srcFileName,
-                                        int compressionLevel)
-{
-    int result;
-    stat_t statbuf;
-    int stat_result = 0;
-
-    DISPLAYLEVEL(6, "FIO_compressFilename_dstFile: opening dst: %s", dstFileName);
-    ress.dstFile = FIO_openDstFile(dstFileName);
-    if (ress.dstFile==NULL) return 1;  /* could not open dstFileName */
-    /* Must ony be added after FIO_openDstFile() succeeds.
-     * Otherwise we may delete the destination file if at already exists, and
-     * the user presses Ctrl-C when asked if they wish to overwrite.
-     */
-    addHandler(dstFileName);
-
-    if (strcmp (srcFileName, stdinmark) && UTIL_getFileStat(srcFileName, &statbuf))
-        stat_result = 1;
-    result = FIO_compressFilename_srcFile(ress, dstFileName, srcFileName, compressionLevel);
-    clearHandler();
-
-    if (fclose(ress.dstFile)) { /* error closing dstFile */
-        DISPLAYLEVEL(1, "zstd: %s: %s \n", dstFileName, strerror(errno));
-        result=1;
-    }
-    if ( (result != 0)  /* operation failure */
-      && strcmp(dstFileName, nulmark)      /* special case : don't remove() /dev/null */
-      && strcmp(dstFileName, stdoutmark) ) /* special case : don't remove() stdout */
-        FIO_remove(dstFileName); /* remove compression artefact; note don't do anything special if remove() fails */
-    else if ( strcmp(dstFileName, stdoutmark)
-           && strcmp(dstFileName, nulmark)
-           && stat_result)
-        UTIL_setFileStat(dstFileName, &statbuf);
-
-    return result;
-}
-
-
 int FIO_compressFilename(const char* dstFileName, const char* srcFileName,
                          const char* dictFileName, int compressionLevel,
                          ZSTD_compressionParameters comprParams)
@@ -1154,7 +1180,7 @@ int FIO_compressFilename(const char* dstFileName, const char* srcFileName,
     U64 const srcSize = (fileSize == UTIL_FILESIZE_UNKNOWN) ? ZSTD_CONTENTSIZE_UNKNOWN : fileSize;
 
     cRess_t const ress = FIO_createCResources(dictFileName, compressionLevel, srcSize, comprParams);
-    int const result = FIO_compressFilename_dstFile(ress, dstFileName, srcFileName, compressionLevel);
+    int const result = FIO_compressFilename_srcFile(ress, dstFileName, srcFileName, compressionLevel);
 
     double const seconds = (double)(clock() - start) / CLOCKS_PER_SEC;
     DISPLAYLEVEL(4, "Completed in %.2f sec \n", seconds);
@@ -1164,57 +1190,75 @@ int FIO_compressFilename(const char* dstFileName, const char* srcFileName,
 }
 
 
+/* FIO_determineCompressedName() :
+ * create a destination filename for compressed srcFileName.
+ * @return a pointer to it.
+ * This function never returns an error (it may abort() in case of pb)
+ */
+static const char*
+FIO_determineCompressedName(const char* srcFileName, const char* suffix)
+{
+    static size_t dfnbCapacity = 0;
+    static char* dstFileNameBuffer = NULL;   /* using static allocation : this function cannot be multi-threaded */
+
+    size_t const sfnSize = strlen(srcFileName);
+    size_t const suffixSize = strlen(suffix);
+
+    if (dfnbCapacity <= sfnSize+suffixSize+1) {  /* resize name buffer */
+        free(dstFileNameBuffer);
+        dfnbCapacity = sfnSize + suffixSize + 30;
+        dstFileNameBuffer = (char*)malloc(dfnbCapacity);
+        if (!dstFileNameBuffer) {
+            EXM_THROW(30, "zstd: %s", strerror(errno));
+    }   }
+    strncpy(dstFileNameBuffer, srcFileName, sfnSize+1 /* Include null */);
+    strncat(dstFileNameBuffer, suffix, suffixSize);
+
+    return dstFileNameBuffer;
+}
+
+
+/* FIO_compressMultipleFilenames() :
+ * compress nbFiles files
+ * into one destination (outFileName)
+ * or into one file each (outFileName == NULL, but suffix != NULL).
+ */
 int FIO_compressMultipleFilenames(const char** inFileNamesTable, unsigned nbFiles,
                                   const char* outFileName, const char* suffix,
                                   const char* dictFileName, int compressionLevel,
                                   ZSTD_compressionParameters comprParams)
 {
-    int missed_files = 0;
-    size_t dfnSize = FNSPACE;
-    char*  dstFileName = (char*)malloc(FNSPACE);
-    size_t const suffixSize = suffix ? strlen(suffix) : 0;
+    int error = 0;
     U64 const firstFileSize = UTIL_getFileSize(inFileNamesTable[0]);
     U64 const firstSrcSize = (firstFileSize == UTIL_FILESIZE_UNKNOWN) ? ZSTD_CONTENTSIZE_UNKNOWN : firstFileSize;
     U64 const srcSize = (nbFiles != 1) ? ZSTD_CONTENTSIZE_UNKNOWN : firstSrcSize ;
     cRess_t ress = FIO_createCResources(dictFileName, compressionLevel, srcSize, comprParams);
 
     /* init */
-    if (dstFileName==NULL)
-        EXM_THROW(27, "FIO_compressMultipleFilenames : allocation error for dstFileName");
-    if (outFileName == NULL && suffix == NULL)
-        EXM_THROW(28, "FIO_compressMultipleFilenames : dst unknown");  /* should never happen */
+    assert(outFileName != NULL || suffix != NULL);
 
-    /* loop on each file */
-    if (outFileName != NULL) {
-        unsigned u;
+    if (outFileName != NULL) {   /* output into a single destination (stdout typically) */
         ress.dstFile = FIO_openDstFile(outFileName);
-        if (ress.dstFile==NULL) {  /* could not open outFileName */
-            missed_files = nbFiles;
+        if (ress.dstFile == NULL) {  /* could not open outFileName */
+            error = 1;
         } else {
+            unsigned u;
             for (u=0; u $INTOVOID   # --rm should remain silent
 rm tmp
 $ZSTD -f tmp && die "tmp not present : should have failed"
 test ! -f tmp.zst  # tmp.zst should not be created
-$ECHO "test : do not delete destination when source is not present"
+$ECHO "test : -d -f do not delete destination when source is not present"
 touch tmp    # create destination file
 $ZSTD -d -f tmp.zst && die "attempt to decompress a non existing file"
-test -f tmp  # destination file should still be present (test disabled temporarily)
+test -f tmp  # destination file should still be present
+$ECHO "test : -f do not delete destination when source is not present"
+rm tmp         # erase source file
+touch tmp.zst  # create destination file
+$ZSTD -f tmp && die "attempt to compress a non existing file"
+test -f tmp.zst  # destination file should still be present
 rm tmp*
 
 
@@ -824,6 +829,7 @@ roundTripTest -g1M -P50 "1 --single-thread --long=29" " --zstd=wlog=28 --memory=
 $ECHO "\n===>   adaptive mode "
 roundTripTest -g270000000 " --adapt"
 roundTripTest -g27000000 " --adapt=min=1,max=4"
+$ECHO "===>   test: --adapt must fail on incoherent bounds "
 ./datagen > tmp
 $ZSTD -f -vv --adapt=min=10,max=9 tmp && die "--adapt must fail on incoherent bounds"
 
@@ -834,6 +840,7 @@ if [ "$1" != "--test-large-data" ]; then
 fi
 
 
+#############################################################################
 
 $ECHO "\n===>   large files tests "
 

From 8514bd8eb90544401b878286966bbcbe734ec8f0 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Mon, 1 Oct 2018 17:49:18 -0700
Subject: [PATCH 360/372] updated NEWS in anticipation for v1.3.6

---
 NEWS | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/NEWS b/NEWS
index 1553f5ad8..1e26f1b18 100644
--- a/NEWS
+++ b/NEWS
@@ -1,6 +1,14 @@
 v1.3.6
 perf: much faster dictionary builder, by @jenniferliu
-api : reduced DDict size by 2 KB
+perf: faster dictionary compression on small data when using multiple contexts, by @felixhandte
+perf: faster dictionary decompression when using a very large number of dictionaries simultaneously
+cli : fix : does no longer overwrite destination when source does not exist (#1082)
+cli : new command --adapt, for automatic compression level adaptation
+api : fix : block api can be streamed with > 4 GB, reported by @catid
+api : reduced ZSTD_DDict size by 2 KB
+api : minimum negative compression level is defined, and can be queried using ZSTD_minCLevel().
+build: Reading legacy format is limited to v0.5+ by default. Can be changed at compile time using macro ZSTD_LEGACY_SUPPORT.
+doc : zstd_compression_format.md updated to match wording in IETF RFC 8478
 misc: tests/paramgrill, a parameter optimizer, by @GeorgeLu97
 
 v1.3.5

From d98733b37e3117b8be0e11ff697f25ba44d9db6a Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Mon, 1 Oct 2018 17:50:16 -0700
Subject: [PATCH 361/372] restored backtrace on failure

for Linux and Mac OS-X.

Note : the backtraces fires up through a trap
before the sanitizer get a chance to report.
There are situations where the sanitizer report is actually preferable.

It might be good to consider a kind of build macro
which can disable backtrace
when sanitizer is enabled.
---
 programs/zstdcli.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/programs/zstdcli.c b/programs/zstdcli.c
index 6b0c89d46..1545d1cac 100644
--- a/programs/zstdcli.c
+++ b/programs/zstdcli.c
@@ -547,7 +547,7 @@ int main(int argCount, const char* argv[])
     memset(&compressionParams, 0, sizeof(compressionParams));
 
     /* init crash handler */
-    //FIO_addAbortHandler();
+    FIO_addAbortHandler();
 
     /* command switches */
     for (argNb=1; argNb
Date: Tue, 2 Oct 2018 15:59:11 -0700
Subject: [PATCH 362/372] fixed static analyzer warnings

note : for some reason,
scan-build version on my laptop found problems within fastcover.c
that scan-build on travisCI does not flag.

They are, as usual, false positive :
the analyzer does not understand that a table (`offset`) is correctly filled before usage.
---
 lib/dictBuilder/fastcover.c | 249 ++++++++++++++++++++----------------
 programs/fileio.c           |  16 ++-
 2 files changed, 147 insertions(+), 118 deletions(-)

diff --git a/lib/dictBuilder/fastcover.c b/lib/dictBuilder/fastcover.c
index 6ce8c8809..dfee45743 100644
--- a/lib/dictBuilder/fastcover.c
+++ b/lib/dictBuilder/fastcover.c
@@ -245,39 +245,41 @@ static int FASTCOVER_checkParameters(ZDICT_cover_params_t parameters,
 /**
  * Clean up a context initialized with `FASTCOVER_ctx_init()`.
  */
-static void FASTCOVER_ctx_destroy(FASTCOVER_ctx_t *ctx) {
-  if (!ctx) {
-    return;
-  }
+static void
+FASTCOVER_ctx_destroy(FASTCOVER_ctx_t* ctx)
+{
+    if (!ctx) return;
 
-  free(ctx->freqs);
-  ctx->freqs = NULL;
+    free(ctx->freqs);
+    ctx->freqs = NULL;
 
-  free(ctx->offsets);
-  ctx->offsets = NULL;
+    free(ctx->offsets);
+    ctx->offsets = NULL;
 }
 
 
 /**
  * Calculate for frequency of hash value of each dmer in ctx->samples
  */
-static void FASTCOVER_computeFrequency(U32 *freqs, FASTCOVER_ctx_t *ctx){
-  const unsigned f = ctx->f;
-  const unsigned d = ctx->d;
-  const unsigned skip = ctx->accelParams.skip;
-  const unsigned readLength = MAX(d, 8);
-  size_t start; /* start of current dmer */
-  size_t i;
-  for (i = 0; i < ctx->nbTrainSamples; i++) {
-    size_t currSampleStart = ctx->offsets[i];
-    size_t currSampleEnd = ctx->offsets[i+1];
-    start = currSampleStart;
-    while (start + readLength <= currSampleEnd) {
-      const size_t dmerIndex = FASTCOVER_hashPtrToIndex(ctx->samples + start, f, d);
-      freqs[dmerIndex]++;
-      start = start + skip + 1;
+static void
+FASTCOVER_computeFrequency(U32* freqs, const FASTCOVER_ctx_t* ctx)
+{
+    const unsigned f = ctx->f;
+    const unsigned d = ctx->d;
+    const unsigned skip = ctx->accelParams.skip;
+    const unsigned readLength = MAX(d, 8);
+    size_t i;
+    assert(ctx->nbTrainSamples >= 5);
+    assert(ctx->nbTrainSamples <= ctx->nbSamples);
+    for (i = 0; i < ctx->nbTrainSamples; i++) {
+        size_t start = ctx->offsets[i];  /* start of current dmer */
+        size_t const currSampleEnd = ctx->offsets[i+1];
+        while (start + readLength <= currSampleEnd) {
+            const size_t dmerIndex = FASTCOVER_hashPtrToIndex(ctx->samples + start, f, d);
+            freqs[dmerIndex]++;
+            start = start + skip + 1;
+        }
     }
-  }
 }
 
 
@@ -288,84 +290,100 @@ static void FASTCOVER_computeFrequency(U32 *freqs, FASTCOVER_ctx_t *ctx){
  * Returns 1 on success or zero on error.
  * The context must be destroyed with `FASTCOVER_ctx_destroy()`.
  */
-static int FASTCOVER_ctx_init(FASTCOVER_ctx_t *ctx, const void *samplesBuffer,
-                              const size_t *samplesSizes, unsigned nbSamples,
-                              unsigned d, double splitPoint, unsigned f,
-                              FASTCOVER_accel_t accelParams) {
-  const BYTE *const samples = (const BYTE *)samplesBuffer;
-  const size_t totalSamplesSize = COVER_sum(samplesSizes, nbSamples);
-  /* Split samples into testing and training sets */
-  const unsigned nbTrainSamples = splitPoint < 1.0 ? (unsigned)((double)nbSamples * splitPoint) : nbSamples;
-  const unsigned nbTestSamples = splitPoint < 1.0 ? nbSamples - nbTrainSamples : nbSamples;
-  const size_t trainingSamplesSize = splitPoint < 1.0 ? COVER_sum(samplesSizes, nbTrainSamples) : totalSamplesSize;
-  const size_t testSamplesSize = splitPoint < 1.0 ? COVER_sum(samplesSizes + nbTrainSamples, nbTestSamples) : totalSamplesSize;
-  /* Checks */
-  if (totalSamplesSize < MAX(d, sizeof(U64)) ||
-      totalSamplesSize >= (size_t)FASTCOVER_MAX_SAMPLES_SIZE) {
-    DISPLAYLEVEL(1, "Total samples size is too large (%u MB), maximum size is %u MB\n",
-                 (U32)(totalSamplesSize >> 20), (FASTCOVER_MAX_SAMPLES_SIZE >> 20));
-    return 0;
-  }
-  /* Check if there are at least 5 training samples */
-  if (nbTrainSamples < 5) {
-    DISPLAYLEVEL(1, "Total number of training samples is %u and is invalid\n", nbTrainSamples);
-    return 0;
-  }
-  /* Check if there's testing sample */
-  if (nbTestSamples < 1) {
-    DISPLAYLEVEL(1, "Total number of testing samples is %u and is invalid.\n", nbTestSamples);
-    return 0;
-  }
-  /* Zero the context */
-  memset(ctx, 0, sizeof(*ctx));
-  DISPLAYLEVEL(2, "Training on %u samples of total size %u\n", nbTrainSamples,
-               (U32)trainingSamplesSize);
-  DISPLAYLEVEL(2, "Testing on %u samples of total size %u\n", nbTestSamples,
-               (U32)testSamplesSize);
+static int
+FASTCOVER_ctx_init(FASTCOVER_ctx_t* ctx,
+                   const void* samplesBuffer,
+                   const size_t* samplesSizes, unsigned nbSamples,
+                   unsigned d, double splitPoint, unsigned f,
+                   FASTCOVER_accel_t accelParams)
+{
+    const BYTE* const samples = (const BYTE*)samplesBuffer;
+    const size_t totalSamplesSize = COVER_sum(samplesSizes, nbSamples);
+    /* Split samples into testing and training sets */
+    const unsigned nbTrainSamples = splitPoint < 1.0 ? (unsigned)((double)nbSamples * splitPoint) : nbSamples;
+    const unsigned nbTestSamples = splitPoint < 1.0 ? nbSamples - nbTrainSamples : nbSamples;
+    const size_t trainingSamplesSize = splitPoint < 1.0 ? COVER_sum(samplesSizes, nbTrainSamples) : totalSamplesSize;
+    const size_t testSamplesSize = splitPoint < 1.0 ? COVER_sum(samplesSizes + nbTrainSamples, nbTestSamples) : totalSamplesSize;
 
-  ctx->samples = samples;
-  ctx->samplesSizes = samplesSizes;
-  ctx->nbSamples = nbSamples;
-  ctx->nbTrainSamples = nbTrainSamples;
-  ctx->nbTestSamples = nbTestSamples;
-  ctx->nbDmers = trainingSamplesSize - MAX(d, sizeof(U64)) + 1;
-  ctx->d = d;
-  ctx->f = f;
-  ctx->accelParams = accelParams;
-
-  /* The offsets of each file */
-  ctx->offsets = (size_t *)malloc((nbSamples + 1) * sizeof(size_t));
-  if (!ctx->offsets) {
-    DISPLAYLEVEL(1, "Failed to allocate scratch buffers\n");
-    FASTCOVER_ctx_destroy(ctx);
-    return 0;
-  }
-
-  /* Fill offsets from the samplesSizes */
-  {
-    U32 i;
-    ctx->offsets[0] = 0;
-    for (i = 1; i <= nbSamples; ++i) {
-      ctx->offsets[i] = ctx->offsets[i - 1] + samplesSizes[i - 1];
+    /* Checks */
+    if (totalSamplesSize < MAX(d, sizeof(U64)) ||
+        totalSamplesSize >= (size_t)FASTCOVER_MAX_SAMPLES_SIZE) {
+        DISPLAYLEVEL(1, "Total samples size is too large (%u MB), maximum size is %u MB\n",
+                    (U32)(totalSamplesSize >> 20), (FASTCOVER_MAX_SAMPLES_SIZE >> 20));
+        return 0;
     }
-  }
 
-  /* Initialize frequency array of size 2^f */
-  ctx->freqs = (U32 *)calloc(((U64)1 << f), sizeof(U32));
+    /* Check if there are at least 5 training samples */
+    if (nbTrainSamples < 5) {
+        DISPLAYLEVEL(1, "Total number of training samples is %u and is invalid\n", nbTrainSamples);
+        return 0;
+    }
 
-  DISPLAYLEVEL(2, "Computing frequencies\n");
-  FASTCOVER_computeFrequency(ctx->freqs, ctx);
+    /* Check if there's testing sample */
+    if (nbTestSamples < 1) {
+        DISPLAYLEVEL(1, "Total number of testing samples is %u and is invalid.\n", nbTestSamples);
+        return 0;
+    }
 
-  return 1;
+    /* Zero the context */
+    memset(ctx, 0, sizeof(*ctx));
+    DISPLAYLEVEL(2, "Training on %u samples of total size %u\n", nbTrainSamples,
+                    (U32)trainingSamplesSize);
+    DISPLAYLEVEL(2, "Testing on %u samples of total size %u\n", nbTestSamples,
+                    (U32)testSamplesSize);
+
+    ctx->samples = samples;
+    ctx->samplesSizes = samplesSizes;
+    ctx->nbSamples = nbSamples;
+    ctx->nbTrainSamples = nbTrainSamples;
+    ctx->nbTestSamples = nbTestSamples;
+    ctx->nbDmers = trainingSamplesSize - MAX(d, sizeof(U64)) + 1;
+    ctx->d = d;
+    ctx->f = f;
+    ctx->accelParams = accelParams;
+
+    /* The offsets of each file */
+    ctx->offsets = (size_t*)calloc((nbSamples + 1), sizeof(size_t));
+    if (ctx->offsets == NULL) {
+        DISPLAYLEVEL(1, "Failed to allocate scratch buffers \n");
+        FASTCOVER_ctx_destroy(ctx);
+        return 0;
+    }
+
+    /* Fill offsets from the samplesSizes */
+    {   U32 i;
+        ctx->offsets[0] = 0;
+        assert(nbSamples >= 5);
+        for (i = 1; i <= nbSamples; ++i) {
+            ctx->offsets[i] = ctx->offsets[i - 1] + samplesSizes[i - 1];
+        }
+    }
+
+    /* Initialize frequency array of size 2^f */
+    ctx->freqs = (U32*)calloc(((U64)1 << f), sizeof(U32));
+    if (ctx->freqs == NULL) {
+        DISPLAYLEVEL(1, "Failed to allocate frequency table \n");
+        FASTCOVER_ctx_destroy(ctx);
+        return 0;
+    }
+
+    DISPLAYLEVEL(2, "Computing frequencies\n");
+    FASTCOVER_computeFrequency(ctx->freqs, ctx);
+
+    return 1;
 }
 
 
 /**
  * Given the prepared context build the dictionary.
  */
-static size_t FASTCOVER_buildDictionary(const FASTCOVER_ctx_t *ctx, U32 *freqs,
-                                        void *dictBuffer, size_t dictBufferCapacity,
-                                        ZDICT_cover_params_t parameters, U16* segmentFreqs){
+static size_t
+FASTCOVER_buildDictionary(const FASTCOVER_ctx_t* ctx,
+                          U32* freqs,
+                          void* dictBuffer, size_t dictBufferCapacity,
+                          ZDICT_cover_params_t parameters,
+                          U16* segmentFreqs)
+{
   BYTE *const dict = (BYTE *)dictBuffer;
   size_t tail = dictBufferCapacity;
   /* Divide the data up into epochs of equal size.
@@ -416,10 +434,10 @@ static size_t FASTCOVER_buildDictionary(const FASTCOVER_ctx_t *ctx, U32 *freqs,
  * Parameters for FASTCOVER_tryParameters().
  */
 typedef struct FASTCOVER_tryParameters_data_s {
-  const FASTCOVER_ctx_t *ctx;
-  COVER_best_t *best;
-  size_t dictBufferCapacity;
-  ZDICT_cover_params_t parameters;
+    const FASTCOVER_ctx_t* ctx;
+    COVER_best_t* best;
+    size_t dictBufferCapacity;
+    ZDICT_cover_params_t parameters;
 } FASTCOVER_tryParameters_data_t;
 
 
@@ -428,7 +446,8 @@ typedef struct FASTCOVER_tryParameters_data_s {
  * This function is thread safe if zstd is compiled with multithreaded support.
  * It takes its parameters as an *OWNING* opaque pointer to support threading.
  */
-static void FASTCOVER_tryParameters(void *opaque) {
+static void FASTCOVER_tryParameters(void *opaque)
+{
   /* Save parameters as local variables */
   FASTCOVER_tryParameters_data_t *const data = (FASTCOVER_tryParameters_data_t *)opaque;
   const FASTCOVER_ctx_t *const ctx = data->ctx;
@@ -447,8 +466,7 @@ static void FASTCOVER_tryParameters(void *opaque) {
   /* Copy the frequencies because we need to modify them */
   memcpy(freqs, ctx->freqs, ((U64)1 << ctx->f) * sizeof(U32));
   /* Build the dictionary */
-  {
-    const size_t tail = FASTCOVER_buildDictionary(ctx, freqs, dict, dictBufferCapacity,
+  { const size_t tail = FASTCOVER_buildDictionary(ctx, freqs, dict, dictBufferCapacity,
                                                   parameters, segmentFreqs);
     const unsigned nbFinalizeSamples = (unsigned)(ctx->nbTrainSamples * ctx->accelParams.finalize / 100);
     dictBufferCapacity = ZDICT_finalizeDictionary(
@@ -474,9 +492,10 @@ _cleanup:
 }
 
 
-
-static void FASTCOVER_convertToCoverParams(ZDICT_fastCover_params_t fastCoverParams,
-                                          ZDICT_cover_params_t *coverParams) {
+static void
+FASTCOVER_convertToCoverParams(ZDICT_fastCover_params_t fastCoverParams,
+                               ZDICT_cover_params_t* coverParams)
+{
     coverParams->k = fastCoverParams.k;
     coverParams->d = fastCoverParams.d;
     coverParams->steps = fastCoverParams.steps;
@@ -486,9 +505,11 @@ static void FASTCOVER_convertToCoverParams(ZDICT_fastCover_params_t fastCoverPar
 }
 
 
-static void FASTCOVER_convertToFastCoverParams(ZDICT_cover_params_t coverParams,
-                                          ZDICT_fastCover_params_t *fastCoverParams,
-                                          unsigned f, unsigned accel) {
+static void
+FASTCOVER_convertToFastCoverParams(ZDICT_cover_params_t coverParams,
+                                   ZDICT_fastCover_params_t* fastCoverParams,
+                                   unsigned f, unsigned accel)
+{
     fastCoverParams->k = coverParams.k;
     fastCoverParams->d = coverParams.d;
     fastCoverParams->steps = coverParams.steps;
@@ -500,9 +521,12 @@ static void FASTCOVER_convertToFastCoverParams(ZDICT_cover_params_t coverParams,
 }
 
 
-ZDICTLIB_API size_t ZDICT_trainFromBuffer_fastCover(
-    void *dictBuffer, size_t dictBufferCapacity, const void *samplesBuffer,
-    const size_t *samplesSizes, unsigned nbSamples, ZDICT_fastCover_params_t parameters) {
+ZDICTLIB_API size_t
+ZDICT_trainFromBuffer_fastCover(void* dictBuffer, size_t dictBufferCapacity,
+                                const void* samplesBuffer,
+                                const size_t* samplesSizes, unsigned nbSamples,
+                                ZDICT_fastCover_params_t parameters)
+{
     BYTE* const dict = (BYTE*)dictBuffer;
     FASTCOVER_ctx_t ctx;
     ZDICT_cover_params_t coverParams;
@@ -562,10 +586,13 @@ ZDICTLIB_API size_t ZDICT_trainFromBuffer_fastCover(
 }
 
 
-ZDICTLIB_API size_t ZDICT_optimizeTrainFromBuffer_fastCover(
-    void *dictBuffer, size_t dictBufferCapacity, const void *samplesBuffer,
-    const size_t *samplesSizes, unsigned nbSamples,
-    ZDICT_fastCover_params_t *parameters) {
+ZDICTLIB_API size_t
+ZDICT_optimizeTrainFromBuffer_fastCover(
+                    void* dictBuffer, size_t dictBufferCapacity,
+                    const void* samplesBuffer,
+                    const size_t* samplesSizes, unsigned nbSamples,
+                    ZDICT_fastCover_params_t* parameters)
+{
     ZDICT_cover_params_t coverParams;
     FASTCOVER_accel_t accelParams;
     /* constants */
diff --git a/programs/fileio.c b/programs/fileio.c
index b6acb3e16..b66426b1d 100644
--- a/programs/fileio.c
+++ b/programs/fileio.c
@@ -20,10 +20,12 @@
 #  define _POSIX_SOURCE 1          /* disable %llu warnings with MinGW on Windows */
 #endif
 
-#if  defined(__linux__) || (defined(__APPLE__) && defined(__MACH__))
+#if !defined(BACKTRACES_ENABLE) && \
+   (defined(__linux__) || (defined(__APPLE__) && defined(__MACH__)) )
 #  define BACKTRACES_ENABLE 1
 #endif
 
+
 /*-*************************************
 *  Includes
 ***************************************/
@@ -32,6 +34,7 @@
 #include       /* fprintf, fopen, fread, _fileno, stdin, stdout */
 #include      /* malloc, free */
 #include      /* strcmp, strlen */
+#include 
 #include       /* errno */
 #include 
 #ifdef BACKTRACES_ENABLE
@@ -43,10 +46,8 @@
 #  include 
 #endif
 
-#include "debug.h"
-#include "mem.h"
+#include "mem.h"       /* U32, U64 */
 #include "fileio.h"
-#include "util.h"
 
 #define ZSTD_STATIC_LINKING_ONLY   /* ZSTD_magicNumber, ZSTD_frameHeaderSize_max */
 #include "zstd.h"
@@ -113,7 +114,7 @@ static UTIL_time_t g_displayClock = UTIL_TIME_INITIALIZER;
 #define EXM_THROW(error, ...)                                             \
 {                                                                         \
     DISPLAYLEVEL(1, "zstd: ");                                            \
-    DEBUGLOG(1, "Error defined at %s, line %i : \n", __FILE__, __LINE__); \
+    DISPLAYLEVEL(5, "Error defined at %s, line %i : \n", __FILE__, __LINE__); \
     DISPLAYLEVEL(1, "error %i : ", error);                                \
     DISPLAYLEVEL(1, __VA_ARGS__);                                         \
     DISPLAYLEVEL(1, " \n");                                               \
@@ -123,7 +124,7 @@ static UTIL_time_t g_displayClock = UTIL_TIME_INITIALIZER;
 #define CHECK_V(v, f)                                \
     v = f;                                           \
     if (ZSTD_isError(v)) {                           \
-        DEBUGLOG(1, "%s \n", #f);                    \
+        DISPLAYLEVEL(5, "%s \n", #f);                \
         EXM_THROW(11, "%s", ZSTD_getErrorName(v));   \
     }
 #define CHECK(f) { size_t err; CHECK_V(err, f); }
@@ -1211,6 +1212,7 @@ FIO_determineCompressedName(const char* srcFileName, const char* suffix)
         if (!dstFileNameBuffer) {
             EXM_THROW(30, "zstd: %s", strerror(errno));
     }   }
+    assert(dstFileNameBuffer != NULL);
     strncpy(dstFileNameBuffer, srcFileName, sfnSize+1 /* Include null */);
     strncat(dstFileNameBuffer, suffix, suffixSize);
 
@@ -1891,7 +1893,6 @@ static int FIO_decompressDstFile(dRess_t ress, FILE* srcFile,
               && transfer_permissions )          /* file permissions correctly extracted from src */
                 UTIL_setFileStat(dstFileName, &statbuf);  /* transfer file permissions from src into dst */
         }
-        signal(SIGINT, SIG_DFL);
     }
 
     return result;
@@ -2013,6 +2014,7 @@ FIO_determineDstName(const char* srcFileName)
     }
 
     /* return dst name == src name truncated from suffix */
+    assert(dstFileNameBuffer != NULL);
     memcpy(dstFileNameBuffer, srcFileName, sfnSize - suffixSize);
     dstFileNameBuffer[sfnSize-suffixSize] = '\0';
     return dstFileNameBuffer;

From 22ddf3523ad0a6fa6941cce06a26022384727e0b Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Tue, 2 Oct 2018 18:20:20 -0700
Subject: [PATCH 363/372] fixed msan warning

on btlazy2 strategy with dictAttach
---
 lib/compress/zstd_lazy.c | 70 +++++++++++++++++++++-------------------
 1 file changed, 36 insertions(+), 34 deletions(-)

diff --git a/lib/compress/zstd_lazy.c b/lib/compress/zstd_lazy.c
index fb9ee3455..4ca69e3ec 100644
--- a/lib/compress/zstd_lazy.c
+++ b/lib/compress/zstd_lazy.c
@@ -16,8 +16,8 @@
 *  Binary Tree search
 ***************************************/
 
-static void ZSTD_updateDUBT(
-                ZSTD_matchState_t* ms,
+static void
+ZSTD_updateDUBT(ZSTD_matchState_t* ms,
                 const BYTE* ip, const BYTE* iend,
                 U32 mls)
 {
@@ -60,8 +60,8 @@ static void ZSTD_updateDUBT(
  *  sort one already inserted but unsorted position
  *  assumption : current >= btlow == (current - btmask)
  *  doesn't fail */
-static void ZSTD_insertDUBT1(
-                 ZSTD_matchState_t* ms,
+static void
+ZSTD_insertDUBT1(ZSTD_matchState_t* ms,
                  U32 current, const BYTE* inputEnd,
                  U32 nbCompares, U32 btLow, const ZSTD_dictMode_e dictMode)
 {
@@ -142,14 +142,15 @@ static void ZSTD_insertDUBT1(
 }
 
 
-static size_t ZSTD_DUBT_findBetterDictMatch (
+static size_t
+ZSTD_DUBT_findBetterDictMatch (
         ZSTD_matchState_t* ms,
         const BYTE* const ip, const BYTE* const iend,
         size_t* offsetPtr,
-        size_t bestLength,
         U32 nbCompares,
         U32 const mls,
-        const ZSTD_dictMode_e dictMode) {
+        const ZSTD_dictMode_e dictMode)
+{
     const ZSTD_matchState_t * const dms = ms->dictMatchState;
     const ZSTD_compressionParameters* const dmsCParams = &dms->cParams;
     const U32 * const dictHashTable = dms->hashTable;
@@ -171,7 +172,7 @@ static size_t ZSTD_DUBT_findBetterDictMatch (
     U32         const btMask = (1 << btLog) - 1;
     U32         const btLow = (btMask >= dictHighLimit - dictLowLimit) ? dictLowLimit : dictHighLimit - btMask;
 
-    size_t commonLengthSmaller=0, commonLengthLarger=0;
+    size_t commonLengthSmaller=0, commonLengthLarger=0, bestLength=0;
     U32 matchEndIdx = current+8+1;
 
     (void)dictMode;
@@ -190,15 +191,16 @@ static size_t ZSTD_DUBT_findBetterDictMatch (
             if (matchLength > matchEndIdx - matchIndex)
                 matchEndIdx = matchIndex + (U32)matchLength;
             if ( (4*(int)(matchLength-bestLength)) > (int)(ZSTD_highbit32(current-matchIndex+1) - ZSTD_highbit32((U32)offsetPtr[0]+1)) ) {
-                DEBUGLOG(9, "ZSTD_DUBT_findBestDictMatch(%u) : found better match length %u -> %u and offsetCode %u -> %u (dictMatchIndex %u, matchIndex %u)",
+                DEBUGLOG(2, "ZSTD_DUBT_findBestDictMatch(%u) : found better match length %u -> %u and offsetCode %u -> %u (dictMatchIndex %u, matchIndex %u)",
                     current, (U32)bestLength, (U32)matchLength, (U32)*offsetPtr, ZSTD_REP_MOVE + current - matchIndex, dictMatchIndex, matchIndex);
                 bestLength = matchLength, *offsetPtr = ZSTD_REP_MOVE + current - matchIndex;
             }
-            if (ip+matchLength == iend) {   /* equal : no way to know if inf or sup */
+            if (ip+matchLength == iend) {   /* reached end of input : ip[matchLength] is not valid, no way to know if it's larger or smaller than match */
                 break;   /* drop, to guarantee consistency (miss a little bit of compression) */
             }
         }
 
+        DEBUGLOG(2, "matchLength:%6zu, match:%p, prefixStart:%p, ip:%p", matchLength, match, prefixStart, ip);
         if (match[matchLength] < ip[matchLength]) {
             if (dictMatchIndex <= btLow) { break; }   /* beyond tree size, stop the search */
             commonLengthSmaller = matchLength;    /* all smaller will now have at least this guaranteed common length */
@@ -213,7 +215,7 @@ static size_t ZSTD_DUBT_findBetterDictMatch (
 
     if (bestLength >= MINMATCH) {
         U32 const mIndex = current - ((U32)*offsetPtr - ZSTD_REP_MOVE); (void)mIndex;
-        DEBUGLOG(8, "ZSTD_DUBT_findBestDictMatch(%u) : found match of length %u and offsetCode %u (pos %u)",
+        DEBUGLOG(2, "ZSTD_DUBT_findBestDictMatch(%u) : found match of length %u and offsetCode %u (pos %u)",
                     current, (U32)bestLength, (U32)*offsetPtr, mIndex);
     }
     return bestLength;
@@ -221,12 +223,12 @@ static size_t ZSTD_DUBT_findBetterDictMatch (
 }
 
 
-static size_t ZSTD_DUBT_findBestMatch (
-                            ZSTD_matchState_t* ms,
-                            const BYTE* const ip, const BYTE* const iend,
-                            size_t* offsetPtr,
-                            U32 const mls,
-                            const ZSTD_dictMode_e dictMode)
+static size_t
+ZSTD_DUBT_findBestMatch(ZSTD_matchState_t* ms,
+                        const BYTE* const ip, const BYTE* const iend,
+                        size_t* offsetPtr,
+                        U32 const mls,
+                        const ZSTD_dictMode_e dictMode)
 {
     const ZSTD_compressionParameters* const cParams = &ms->cParams;
     U32*   const hashTable = ms->hashTable;
@@ -344,7 +346,7 @@ static size_t ZSTD_DUBT_findBestMatch (
         *smallerPtr = *largerPtr = 0;
 
         if (dictMode == ZSTD_dictMatchState && nbCompares) {
-            bestLength = ZSTD_DUBT_findBetterDictMatch(ms, ip, iend, offsetPtr, bestLength, nbCompares, mls, dictMode);
+            bestLength = ZSTD_DUBT_findBetterDictMatch(ms, ip, iend, offsetPtr, nbCompares, mls, dictMode);
         }
 
         assert(matchEndIdx > current+8); /* ensure nextToUpdate is increased */
@@ -360,12 +362,12 @@ static size_t ZSTD_DUBT_findBestMatch (
 
 
 /** ZSTD_BtFindBestMatch() : Tree updater, providing best match */
-FORCE_INLINE_TEMPLATE size_t ZSTD_BtFindBestMatch (
-                        ZSTD_matchState_t* ms,
-                        const BYTE* const ip, const BYTE* const iLimit,
-                        size_t* offsetPtr,
-                        const U32 mls /* template */,
-                        const ZSTD_dictMode_e dictMode)
+FORCE_INLINE_TEMPLATE size_t
+ZSTD_BtFindBestMatch( ZSTD_matchState_t* ms,
+                const BYTE* const ip, const BYTE* const iLimit,
+                      size_t* offsetPtr,
+                const U32 mls /* template */,
+                const ZSTD_dictMode_e dictMode)
 {
     DEBUGLOG(7, "ZSTD_BtFindBestMatch");
     if (ip < ms->window.base + ms->nextToUpdate) return 0;   /* skipped area */
@@ -374,10 +376,10 @@ FORCE_INLINE_TEMPLATE size_t ZSTD_BtFindBestMatch (
 }
 
 
-static size_t ZSTD_BtFindBestMatch_selectMLS (
-                        ZSTD_matchState_t* ms,
-                        const BYTE* ip, const BYTE* const iLimit,
-                        size_t* offsetPtr)
+static size_t
+ZSTD_BtFindBestMatch_selectMLS (  ZSTD_matchState_t* ms,
+                            const BYTE* ip, const BYTE* const iLimit,
+                                  size_t* offsetPtr)
 {
     switch(ms->cParams.searchLength)
     {
@@ -679,7 +681,7 @@ size_t ZSTD_compressBlock_lazy_generic(
         }
 
         /* first search (depth 0) */
-        {   size_t offsetFound = 99999999;
+        {   size_t offsetFound = 999999999;
             size_t const ml2 = searchMax(ms, ip, iend, &offsetFound);
             if (ml2 > matchLength)
                 matchLength = ml2, start = ip, offset=offsetFound;
@@ -717,7 +719,7 @@ size_t ZSTD_compressBlock_lazy_generic(
                         matchLength = mlRep, offset = 0, start = ip;
                 }
             }
-            {   size_t offset2=99999999;
+            {   size_t offset2=999999999;
                 size_t const ml2 = searchMax(ms, ip, iend, &offset2);
                 int const gain2 = (int)(ml2*4 - ZSTD_highbit32((U32)offset2+1));   /* raw approx */
                 int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offset+1) + 4);
@@ -752,7 +754,7 @@ size_t ZSTD_compressBlock_lazy_generic(
                             matchLength = mlRep, offset = 0, start = ip;
                     }
                 }
-                {   size_t offset2=99999999;
+                {   size_t offset2=999999999;
                     size_t const ml2 = searchMax(ms, ip, iend, &offset2);
                     int const gain2 = (int)(ml2*4 - ZSTD_highbit32((U32)offset2+1));   /* raw approx */
                     int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offset+1) + 7);
@@ -942,7 +944,7 @@ size_t ZSTD_compressBlock_lazy_extDict_generic(
         }   }
 
         /* first search (depth 0) */
-        {   size_t offsetFound = 99999999;
+        {   size_t offsetFound = 999999999;
             size_t const ml2 = searchMax(ms, ip, iend, &offsetFound);
             if (ml2 > matchLength)
                 matchLength = ml2, start = ip, offset=offsetFound;
@@ -975,7 +977,7 @@ size_t ZSTD_compressBlock_lazy_extDict_generic(
             }   }
 
             /* search match, depth 1 */
-            {   size_t offset2=99999999;
+            {   size_t offset2=999999999;
                 size_t const ml2 = searchMax(ms, ip, iend, &offset2);
                 int const gain2 = (int)(ml2*4 - ZSTD_highbit32((U32)offset2+1));   /* raw approx */
                 int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offset+1) + 4);
@@ -1005,7 +1007,7 @@ size_t ZSTD_compressBlock_lazy_extDict_generic(
                 }   }
 
                 /* search match, depth 2 */
-                {   size_t offset2=99999999;
+                {   size_t offset2=999999999;
                     size_t const ml2 = searchMax(ms, ip, iend, &offset2);
                     int const gain2 = (int)(ml2*4 - ZSTD_highbit32((U32)offset2+1));   /* raw approx */
                     int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offset+1) + 7);

From 87c10e2f58cf7b26e0532e0c1cffe3fe4eb9a033 Mon Sep 17 00:00:00 2001
From: Jerome Duval 
Date: Sun, 6 Aug 2017 22:27:54 +0200
Subject: [PATCH 364/372] Enable building zstd on Haiku.

---
 Makefile            | 2 +-
 lib/Makefile        | 2 +-
 programs/Makefile   | 2 +-
 programs/platform.h | 5 +++--
 4 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/Makefile b/Makefile
index a17900450..0ad44a679 100644
--- a/Makefile
+++ b/Makefile
@@ -113,7 +113,7 @@ clean:
 #------------------------------------------------------------------------------
 # make install is validated only for Linux, macOS, Hurd and some BSD targets
 #------------------------------------------------------------------------------
-ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD DragonFly NetBSD MSYS_NT))
+ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD DragonFly NetBSD MSYS_NT Haiku))
 
 HOST_OS = POSIX
 CMAKE_PARAMS = -DZSTD_BUILD_CONTRIB:BOOL=ON -DZSTD_BUILD_STATIC:BOOL=ON -DZSTD_BUILD_TESTS:BOOL=ON -DZSTD_ZLIB_SUPPORT:BOOL=ON -DZSTD_LZMA_SUPPORT:BOOL=ON -DCMAKE_BUILD_TYPE=Release
diff --git a/lib/Makefile b/lib/Makefile
index cf8e45b0f..f3948f2dd 100644
--- a/lib/Makefile
+++ b/lib/Makefile
@@ -157,7 +157,7 @@ clean:
 #-----------------------------------------------------------------------------
 # make install is validated only for Linux, macOS, BSD, Hurd and Solaris targets
 #-----------------------------------------------------------------------------
-ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS))
+ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS Haiku))
 
 DESTDIR     ?=
 # directory variables : GNU conventions prefer lowercase
diff --git a/programs/Makefile b/programs/Makefile
index f1a96325c..5bbc82758 100644
--- a/programs/Makefile
+++ b/programs/Makefile
@@ -258,7 +258,7 @@ preview-man: clean-man man
 #-----------------------------------------------------------------------------
 # make install is validated only for Linux, macOS, BSD, Hurd and Solaris targets
 #-----------------------------------------------------------------------------
-ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS))
+ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS Haiku))
 
 .PHONY: list
 list:
diff --git a/programs/platform.h b/programs/platform.h
index 89eba37ec..aa19ee12f 100644
--- a/programs/platform.h
+++ b/programs/platform.h
@@ -70,9 +70,10 @@ extern "C" {
 *  PLATFORM_POSIX_VERSION >= 1 is equal to found _POSIX_VERSION
 ***************************************************************/
 #if !defined(_WIN32) && (defined(__unix__) || defined(__unix) || (defined(__APPLE__) && defined(__MACH__)) /* UNIX-like OS */ \
-   || defined(__midipix__) || defined(__VMS))
+   || defined(__midipix__) || defined(__VMS) || defined(__HAIKU__))
 #  if (defined(__APPLE__) && defined(__MACH__)) || defined(__SVR4) || defined(_AIX) || defined(__hpux) /* POSIX.1-2001 (SUSv3) conformant */ \
-     || defined(__DragonFly__) || defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__)  /* BSD distros */
+     || defined(__DragonFly__) || defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__)  /* BSD distros */ \
+     || defined(__HAIKU__)
 #    define PLATFORM_POSIX_VERSION 200112L
 #  else
 #    if defined(__linux__) || defined(__linux)

From b1407f9acd83906fbe890ecb6b61de9cec004420 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Wed, 3 Oct 2018 12:43:59 -0700
Subject: [PATCH 365/372] fixed wrong assert() position

could fire on invalid input.
blocking for afl tests.
---
 programs/fileio.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/programs/fileio.c b/programs/fileio.c
index b66426b1d..ed3a29cda 100644
--- a/programs/fileio.c
+++ b/programs/fileio.c
@@ -1449,12 +1449,12 @@ static void FIO_zstdErrorHelp(dRess_t* ress, size_t err, char const* srcFileName
     if (err == 0) {
         unsigned long long const windowSize = header.windowSize;
         U32 const windowLog = FIO_highbit64(windowSize) + ((windowSize & (windowSize - 1)) != 0);
-        U32 const windowMB = (U32)((windowSize >> 20) + ((windowSize & ((1 MB) - 1)) != 0));
-        assert(windowSize < (U64)(1ULL << 52));
         assert(g_memLimit > 0);
         DISPLAYLEVEL(1, "%s : Window size larger than maximum : %llu > %u\n",
                         srcFileName, windowSize, g_memLimit);
         if (windowLog <= ZSTD_WINDOWLOG_MAX) {
+            U32 const windowMB = (U32)((windowSize >> 20) + ((windowSize & ((1 MB) - 1)) != 0));
+            assert(windowSize < (U64)(1ULL << 52));   /* ensure now overflow for windowMB */
             DISPLAYLEVEL(1, "%s : Use --long=%u or --memory=%uMB\n",
                             srcFileName, windowLog, windowMB);
             return;

From 549c19b42ed76f51fc0a928b61bd965f5dc99471 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Wed, 3 Oct 2018 14:54:33 -0700
Subject: [PATCH 366/372] portability macro flags updates, for Haiku

some non-trivial changes to platform.h and util.h,
initially related to compilation for Haiku,
but I used this opportunity to make them cleaner
and add some documentation.

Noticed several tests that could be improved
(too harsh conditions, useless exception, etc.)
but I did not dare modifying too many tests just before release.
---
 programs/platform.h | 58 ++++++++++++++++++++++++++++++++-------------
 programs/util.h     | 39 ++++++++++++++++++------------
 2 files changed, 66 insertions(+), 31 deletions(-)

diff --git a/programs/platform.h b/programs/platform.h
index aa19ee12f..b6b3a3bea 100644
--- a/programs/platform.h
+++ b/programs/platform.h
@@ -65,39 +65,52 @@ extern "C" {
 
 /* ************************************************************
 *  Detect POSIX version
-*  PLATFORM_POSIX_VERSION = -1 for non-Unix e.g. Windows
-*  PLATFORM_POSIX_VERSION = 0 for Unix-like non-POSIX
-*  PLATFORM_POSIX_VERSION >= 1 is equal to found _POSIX_VERSION
+*  PLATFORM_POSIX_VERSION = 0 for non-Unix e.g. Windows
+*  PLATFORM_POSIX_VERSION = 1 for Unix-like but non-POSIX
+*  PLATFORM_POSIX_VERSION > 1 is equal to found _POSIX_VERSION
+*  Value of PLATFORM_POSIX_VERSION can be forced on command line
 ***************************************************************/
-#if !defined(_WIN32) && (defined(__unix__) || defined(__unix) || (defined(__APPLE__) && defined(__MACH__)) /* UNIX-like OS */ \
-   || defined(__midipix__) || defined(__VMS) || defined(__HAIKU__))
+#ifndef PLATFORM_POSIX_VERSION
+
 #  if (defined(__APPLE__) && defined(__MACH__)) || defined(__SVR4) || defined(_AIX) || defined(__hpux) /* POSIX.1-2001 (SUSv3) conformant */ \
-     || defined(__DragonFly__) || defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__)  /* BSD distros */ \
-     || defined(__HAIKU__)
+     || defined(__DragonFly__) || defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__)  /* BSD distros */
+     /* exception rule : force posix version to 200112L,
+      * note: it's better to use unistd.h's _POSIX_VERSION whenever possible */
 #    define PLATFORM_POSIX_VERSION 200112L
-#  else
+
+/* try to determine posix version through official unistd.h's _POSIX_VERSION (http://pubs.opengroup.org/onlinepubs/7908799/xsh/unistd.h.html).
+ * note : there is no simple way to know in advance if  is present or not on target system,
+ * Posix specification mandates its presence and its content, but target system must respect this spec.
+ * It's necessary to _not_ #include  whenever target OS is not unix-like
+ * otherwise it will block preprocessing stage.
+ * The following list of build macros tries to "guess" if target OS is likely unix-like, and therefore can #include 
+ */
+#  elif !defined(_WIN32) \
+     && (defined(__unix__) || defined(__unix) \
+     || defined(__midipix__) || defined(__VMS) || defined(__HAIKU__))
+
 #    if defined(__linux__) || defined(__linux)
 #      ifndef _POSIX_C_SOURCE
-#        define _POSIX_C_SOURCE 200112L  /* use feature test macro */
+#        define _POSIX_C_SOURCE 200112L  /* feature test macro : https://www.gnu.org/software/libc/manual/html_node/Feature-Test-Macros.html */
 #      endif
 #    endif
 #    include   /* declares _POSIX_VERSION */
 #    if defined(_POSIX_VERSION)  /* POSIX compliant */
 #      define PLATFORM_POSIX_VERSION _POSIX_VERSION
 #    else
-#      define PLATFORM_POSIX_VERSION 0
+#      define PLATFORM_POSIX_VERSION 1
 #    endif
-#  endif
-#endif
-#if !defined(PLATFORM_POSIX_VERSION)
-#  define PLATFORM_POSIX_VERSION -1
-#endif
 
+#  else  /* non-unix target platform (like Windows) */
+#    define PLATFORM_POSIX_VERSION 0
+#  endif
+
+#endif   /* PLATFORM_POSIX_VERSION */
 
 /*-*********************************************
 *  Detect if isatty() and fileno() are available
 ************************************************/
-#if (defined(__linux__) && (PLATFORM_POSIX_VERSION >= 1)) \
+#if (defined(__linux__) && (PLATFORM_POSIX_VERSION > 1)) \
  || (PLATFORM_POSIX_VERSION >= 200112L) \
  || defined(__DJGPP__) \
  || defined(__MSYS__)
@@ -160,6 +173,19 @@ static __inline int IS_CONSOLE(FILE* stdStream) {
 #endif
 
 
+#ifndef ZSTD_SETPRIORITY_SUPPORT
+   /* mandates presence of  and support for setpriority() : http://man7.org/linux/man-pages/man2/setpriority.2.html */
+#  define ZSTD_SETPRIORITY_SUPPORT (PLATFORM_POSIX_VERSION >= 200112L)
+#endif
+
+
+#ifndef ZSTD_NANOSLEEP_SUPPORT
+   /* mandates support of nanosleep() within  : http://man7.org/linux/man-pages/man2/nanosleep.2.html */
+#  define ZSTD_NANOSLEEP_SUPPORT (defined(__linux__) && (PLATFORM_POSIX_VERSION >= 199309L)) \
+                              || (PLATFORM_POSIX_VERSION >= 200112L)
+#endif
+
+
 #if defined (__cplusplus)
 }
 #endif
diff --git a/programs/util.h b/programs/util.h
index d6184fac8..596f4bbbc 100644
--- a/programs/util.h
+++ b/programs/util.h
@@ -26,7 +26,7 @@ extern "C" {
 #include         /* fprintf */
 #include        /* strncmp */
 #include     /* stat, utime */
-#include      /* stat */
+#include      /* stat, chmod */
 #if defined(_MSC_VER)
 #  include   /* utime */
 #  include          /* _chmod */
@@ -53,32 +53,34 @@ extern "C" {
 #endif
 
 
-/*-****************************************
-*  Sleep functions: Windows - Posix - others
-******************************************/
+/*-*************************************************
+*  Sleep & priority functions: Windows - Posix - others
+***************************************************/
 #if defined(_WIN32)
 #  include 
 #  define SET_REALTIME_PRIORITY SetPriorityClass(GetCurrentProcess(), REALTIME_PRIORITY_CLASS)
 #  define UTIL_sleep(s) Sleep(1000*s)
 #  define UTIL_sleepMilli(milli) Sleep(milli)
-#elif PLATFORM_POSIX_VERSION >= 0 /* Unix-like operating system */
-#  include 
-#  include  /* setpriority */
-#  if defined(PRIO_PROCESS)
-#    define SET_REALTIME_PRIORITY setpriority(PRIO_PROCESS, 0, -20)
-#  else
-#    define SET_REALTIME_PRIORITY /* disabled */
-#  endif
+
+#elif PLATFORM_POSIX_VERSION > 0 /* Unix-like operating system */
+#  include    /* sleep */
 #  define UTIL_sleep(s) sleep(s)
-#  if (defined(__linux__) && (PLATFORM_POSIX_VERSION >= 199309L)) || (PLATFORM_POSIX_VERSION >= 200112L)  /* nanosleep requires POSIX.1-2001 */
+#  if ZSTD_NANOSLEEP_SUPPORT
 #      define UTIL_sleepMilli(milli) { struct timespec t; t.tv_sec=0; t.tv_nsec=milli*1000000ULL; nanosleep(&t, NULL); }
 #  else
 #      define UTIL_sleepMilli(milli) /* disabled */
 #  endif
-#else
-#  define SET_REALTIME_PRIORITY      /* disabled */
+#  if ZSTD_SETPRIORITY_SUPPORT
+#    include  /* setpriority */
+#    define SET_REALTIME_PRIORITY setpriority(PRIO_PROCESS, 0, -20)
+#  else
+#    define SET_REALTIME_PRIORITY /* disabled */
+#  endif
+
+#else  /* unknown non-unix operating systen */
 #  define UTIL_sleep(s)          /* disabled */
 #  define UTIL_sleepMilli(milli) /* disabled */
+#  define SET_REALTIME_PRIORITY  /* disabled */
 #endif
 
 
@@ -119,6 +121,7 @@ static int g_utilDisplayLevel;
 #if defined(_WIN32)   /* Windows */
     #define UTIL_TIME_INITIALIZER { { 0, 0 } }
     typedef LARGE_INTEGER UTIL_time_t;
+
     UTIL_STATIC UTIL_time_t UTIL_getTime(void) { UTIL_time_t x; QueryPerformanceCounter(&x); return x; }
     UTIL_STATIC U64 UTIL_getSpanTimeMicro(UTIL_time_t clockStart, UTIL_time_t clockEnd)
     {
@@ -148,6 +151,7 @@ static int g_utilDisplayLevel;
     #include 
     #define UTIL_TIME_INITIALIZER 0
     typedef U64 UTIL_time_t;
+
     UTIL_STATIC UTIL_time_t UTIL_getTime(void) { return mach_absolute_time(); }
     UTIL_STATIC U64 UTIL_getSpanTimeMicro(UTIL_time_t clockStart, UTIL_time_t clockEnd)
     {
@@ -179,6 +183,7 @@ static int g_utilDisplayLevel;
     #define UTIL_TIME_INITIALIZER { 0, 0 }
     typedef struct timespec UTIL_freq_t;
     typedef struct timespec UTIL_time_t;
+
     UTIL_STATIC UTIL_time_t UTIL_getTime(void)
     {
         UTIL_time_t time;
@@ -186,6 +191,7 @@ static int g_utilDisplayLevel;
             UTIL_DISPLAYLEVEL(1, "ERROR: Failed to get time\n");   /* we could also exit() */
         return time;
     }
+
     UTIL_STATIC UTIL_time_t UTIL_getSpanTime(UTIL_time_t begin, UTIL_time_t end)
     {
         UTIL_time_t diff;
@@ -198,6 +204,7 @@ static int g_utilDisplayLevel;
         }
         return diff;
     }
+
     UTIL_STATIC U64 UTIL_getSpanTimeMicro(UTIL_time_t begin, UTIL_time_t end)
     {
         UTIL_time_t const diff = UTIL_getSpanTime(begin, end);
@@ -206,6 +213,7 @@ static int g_utilDisplayLevel;
         micro += diff.tv_nsec / 1000ULL;
         return micro;
     }
+
     UTIL_STATIC U64 UTIL_getSpanTimeNano(UTIL_time_t begin, UTIL_time_t end)
     {
         UTIL_time_t const diff = UTIL_getSpanTime(begin, end);
@@ -214,6 +222,7 @@ static int g_utilDisplayLevel;
         nano += diff.tv_nsec;
         return nano;
     }
+
 #else   /* relies on standard C (note : clock_t measurements can be wrong when using multi-threading) */
     typedef clock_t UTIL_time_t;
     #define UTIL_TIME_INITIALIZER 0

From 4a85b126d910e2867cd1a83c64f6291f1a4a7694 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Wed, 3 Oct 2018 15:34:41 -0700
Subject: [PATCH 367/372] changed ZSTD_NANOSLEEP_SUPPORT definition

to please `-Wexpansion-to-defined`
---
 programs/platform.h | 18 +++++++++++-------
 programs/util.h     |  4 ++--
 2 files changed, 13 insertions(+), 9 deletions(-)

diff --git a/programs/platform.h b/programs/platform.h
index b6b3a3bea..155ebcd1e 100644
--- a/programs/platform.h
+++ b/programs/platform.h
@@ -50,15 +50,15 @@ extern "C" {
 /* *********************************************************
 *  Turn on Large Files support (>4GB) for 32-bit Linux/Unix
 ***********************************************************/
-#if !defined(__64BIT__) || defined(__MINGW32__)       /* No point defining Large file for 64 bit but MinGW-w64 requires it */
+#if !defined(__64BIT__) || defined(__MINGW32__)    /* No point defining Large file for 64 bit but MinGW-w64 requires it */
 #  if !defined(_FILE_OFFSET_BITS)
-#    define _FILE_OFFSET_BITS 64                      /* turn off_t into a 64-bit type for ftello, fseeko */
+#    define _FILE_OFFSET_BITS 64                   /* turn off_t into a 64-bit type for ftello, fseeko */
 #  endif
-#  if !defined(_LARGEFILE_SOURCE)                     /* obsolete macro, replaced with _FILE_OFFSET_BITS */
-#    define _LARGEFILE_SOURCE 1                       /* Large File Support extension (LFS) - fseeko, ftello */
+#  if !defined(_LARGEFILE_SOURCE)                  /* obsolete macro, replaced with _FILE_OFFSET_BITS */
+#    define _LARGEFILE_SOURCE 1                    /* Large File Support extension (LFS) - fseeko, ftello */
 #  endif
 #  if defined(_AIX) || defined(__hpux)
-#    define _LARGE_FILES                              /* Large file support on 32-bits AIX and HP-UX */
+#    define _LARGE_FILES                           /* Large file support on 32-bits AIX and HP-UX */
 #  endif
 #endif
 
@@ -181,8 +181,12 @@ static __inline int IS_CONSOLE(FILE* stdStream) {
 
 #ifndef ZSTD_NANOSLEEP_SUPPORT
    /* mandates support of nanosleep() within  : http://man7.org/linux/man-pages/man2/nanosleep.2.html */
-#  define ZSTD_NANOSLEEP_SUPPORT (defined(__linux__) && (PLATFORM_POSIX_VERSION >= 199309L)) \
-                              || (PLATFORM_POSIX_VERSION >= 200112L)
+#  if (defined(__linux__) && (PLATFORM_POSIX_VERSION >= 199309L)) \
+   || (PLATFORM_POSIX_VERSION >= 200112L)
+#     define ZSTD_NANOSLEEP_SUPPORT 1
+#  else
+#     define ZSTD_NANOSLEEP_SUPPORT 0
+#  endif
 #endif
 
 
diff --git a/programs/util.h b/programs/util.h
index 596f4bbbc..67aa7a56b 100644
--- a/programs/util.h
+++ b/programs/util.h
@@ -20,7 +20,7 @@ extern "C" {
 /*-****************************************
 *  Dependencies
 ******************************************/
-#include "platform.h"     /* PLATFORM_POSIX_VERSION */
+#include "platform.h"     /* PLATFORM_POSIX_VERSION, ZSTD_NANOSLEEP_SUPPORT, ZSTD_SETPRIORITY_SUPPORT */
 #include        /* malloc */
 #include        /* size_t, ptrdiff_t */
 #include         /* fprintf */
@@ -65,7 +65,7 @@ extern "C" {
 #elif PLATFORM_POSIX_VERSION > 0 /* Unix-like operating system */
 #  include    /* sleep */
 #  define UTIL_sleep(s) sleep(s)
-#  if ZSTD_NANOSLEEP_SUPPORT
+#  if ZSTD_NANOSLEEP_SUPPORT   /* necessarily defined in platform.h */
 #      define UTIL_sleepMilli(milli) { struct timespec t; t.tv_sec=0; t.tv_nsec=milli*1000000ULL; nanosleep(&t, NULL); }
 #  else
 #      define UTIL_sleepMilli(milli) /* disabled */

From 11cd2ea43da644cf3f2bcc088e183290619e9cf7 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Wed, 3 Oct 2018 16:37:50 -0700
Subject: [PATCH 368/372] finalized minor warnings on Haiku

---
 NEWS                  |  3 ++-
 lib/legacy/zstd_v05.c | 10 +++++-----
 2 files changed, 7 insertions(+), 6 deletions(-)

diff --git a/NEWS b/NEWS
index 1e26f1b18..40805c187 100644
--- a/NEWS
+++ b/NEWS
@@ -7,7 +7,8 @@ cli : new command --adapt, for automatic compression level adaptation
 api : fix : block api can be streamed with > 4 GB, reported by @catid
 api : reduced ZSTD_DDict size by 2 KB
 api : minimum negative compression level is defined, and can be queried using ZSTD_minCLevel().
-build: Reading legacy format is limited to v0.5+ by default. Can be changed at compile time using macro ZSTD_LEGACY_SUPPORT.
+build: support Haiku target, by @korli
+build: Read Legacy format is limited to v0.5+ by default. Can be changed at compile time with macro ZSTD_LEGACY_SUPPORT.
 doc : zstd_compression_format.md updated to match wording in IETF RFC 8478
 misc: tests/paramgrill, a parameter optimizer, by @GeorgeLu97
 
diff --git a/lib/legacy/zstd_v05.c b/lib/legacy/zstd_v05.c
index 6f4dc72b7..a1580a271 100644
--- a/lib/legacy/zstd_v05.c
+++ b/lib/legacy/zstd_v05.c
@@ -3300,11 +3300,11 @@ static size_t ZSTDv05_decompressSequences(
     BYTE* const ostart = (BYTE* const)dst;
     BYTE* op = ostart;
     BYTE* const oend = ostart + maxDstSize;
-    size_t errorCode, dumpsLength;
+    size_t errorCode, dumpsLength=0;
     const BYTE* litPtr = dctx->litPtr;
     const BYTE* const litEnd = litPtr + dctx->litSize;
-    int nbSeq;
-    const BYTE* dumps;
+    int nbSeq=0;
+    const BYTE* dumps = NULL;
     U32* DTableLL = dctx->LLTable;
     U32* DTableML = dctx->MLTable;
     U32* DTableOffb = dctx->OffTable;
@@ -3413,10 +3413,10 @@ static size_t ZSTDv05_decompress_continueDCtx(ZSTDv05_DCtx* dctx,
     BYTE* const oend = ostart + maxDstSize;
     size_t remainingSize = srcSize;
     blockProperties_t blockProperties;
+    memset(&blockProperties, 0, sizeof(blockProperties));
 
     /* Frame Header */
-    {
-        size_t frameHeaderSize;
+    {   size_t frameHeaderSize;
         if (srcSize < ZSTDv05_frameHeaderSize_min+ZSTDv05_blockHeaderSize) return ERROR(srcSize_wrong);
         frameHeaderSize = ZSTDv05_decodeFrameHeader_Part1(dctx, src, ZSTDv05_frameHeaderSize_min);
         if (ZSTDv05_isError(frameHeaderSize)) return frameHeaderSize;

From 9ac8f2d7b90592f7dbef1a1ac03bf2cb2bdb999a Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Wed, 3 Oct 2018 18:14:35 -0700
Subject: [PATCH 369/372] fixed VS2017Community build script

reported by @epicabsol
---
 build/VS_scripts/build.generic.cmd | 2 +-
 build/cmake/.gitignore             | 3 +++
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/build/VS_scripts/build.generic.cmd b/build/VS_scripts/build.generic.cmd
index bae42fd7f..a7ca4d067 100644
--- a/build/VS_scripts/build.generic.cmd
+++ b/build/VS_scripts/build.generic.cmd
@@ -34,7 +34,7 @@ SET msbuild_vs2017professional="%programfiles(x86)%\Microsoft Visual Studio\2017
 SET msbuild_vs2017enterprise="%programfiles(x86)%\Microsoft Visual Studio\2017\Enterprise\MSBuild\15.0\Bin\MSBuild.exe"
 IF %msbuild_version% == VS2013 SET msbuild="%programfiles(x86)%\MSBuild\12.0\Bin\MSBuild.exe"
 IF %msbuild_version% == VS2015 SET msbuild="%programfiles(x86)%\MSBuild\14.0\Bin\MSBuild.exe"
-IF %msbuild_version% == VS2017Community SET msbuild="%msbuild_vs2017community%
+IF %msbuild_version% == VS2017Community SET msbuild=%msbuild_vs2017community%
 IF %msbuild_version% == VS2017Professional SET msbuild=%msbuild_vs2017professional%
 IF %msbuild_version% == VS2017Enterprise SET msbuild=%msbuild_vs2017enterprise%
 IF %msbuild_version% == VS2017 (
diff --git a/build/cmake/.gitignore b/build/cmake/.gitignore
index ad4283f99..2e51e8954 100644
--- a/build/cmake/.gitignore
+++ b/build/cmake/.gitignore
@@ -1,3 +1,6 @@
+# cmake working directory
+cmakeBuild
+
 # cmake artefacts
 CMakeCache.txt
 CMakeFiles

From efbc3e823d21d23cfd942fabdc65cbb10c5a6571 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Thu, 4 Oct 2018 14:27:13 -0700
Subject: [PATCH 370/372] fixed paramgrill wrong assert() conditions

and slightly refactored affected function.

Honestly, the formula calculating variance should get a second reviewing round,
it's not clear if it's correct.
---
 programs/bench.h   |  2 +-
 tests/paramgrill.c | 42 ++++++++++++++++++++----------------------
 2 files changed, 21 insertions(+), 23 deletions(-)

diff --git a/programs/bench.h b/programs/bench.h
index 59f415896..13ca5b50b 100644
--- a/programs/bench.h
+++ b/programs/bench.h
@@ -45,7 +45,7 @@ typedef struct {
     size_t cSize;
     unsigned long long cSpeed;   /* bytes / sec */
     unsigned long long dSpeed;
-    size_t cMem;                 /* ? what is reported ? */
+    size_t cMem;                 /* memory usage during compression */
 } BMK_benchResult_t;
 
 VARIANT_ERROR_RESULT(BMK_benchResult_t, BMK_benchOutcome_t);
diff --git a/tests/paramgrill.c b/tests/paramgrill.c
index 466a156d3..7a4be854a 100644
--- a/tests/paramgrill.c
+++ b/tests/paramgrill.c
@@ -1552,39 +1552,36 @@ static int allBench(BMK_benchResult_t* resultPtr,
                 BMK_benchResult_t* winnerResult, int feas)
 {
     BMK_benchResult_t benchres;
-    U64 loopDurationC = 0, loopDurationD = 0;
     double uncertaintyConstantC = 3., uncertaintyConstantD = 3.;
     double winnerRS;
 
-    /* initial benchmarking, gives exact ratio and memory, warms up future runs */
-    CBENCHMARK(1, benchres, tmp, BMK_both, 2);
+    BMK_benchOutcome_t const outcome = BMK_benchMemInvertible(buf, ctx, BASE_CLEVEL, &cParams, BMK_both, 2);
+    if (!BMK_isSuccessful_benchOutcome(outcome)) {
+        DEBUGOUTPUT("Benchmarking failed \n");
+        return ERROR_RESULT;
+    }
+    benchres = BMK_extract_benchResult(outcome);
 
     winnerRS = resultScore(*winnerResult, buf.srcSize, target);
-    DEBUGOUTPUT("WinnerScore: %f\n ", winnerRS);
+    DEBUGOUTPUT("WinnerScore: %f \n ", winnerRS);
 
     *resultPtr = benchres;
 
-    /* calculate uncertainty in compression / decompression runs */
-    if(benchres.cSpeed) {
-        loopDurationC = (((U64)buf.srcSize * TIMELOOP_NANOSEC) / benchres.cSpeed);
-        uncertaintyConstantC = ((loopDurationC + (double)(2 * g_clockGranularity))/loopDurationC);
-    }
-
-    if(benchres.dSpeed) {
-        loopDurationD = (((U64)buf.srcSize * TIMELOOP_NANOSEC) / benchres.dSpeed);
-        uncertaintyConstantD = ((loopDurationD + (double)(2 * g_clockGranularity))/loopDurationD);
-    }
-
     /* anything with worse ratio in feas is definitely worse, discard */
     if(feas && benchres.cSize < winnerResult->cSize && !g_optmode) {
         return WORSE_RESULT;
     }
 
-    /* ensure all measurements last a minimum time, to reduce measurement errors */
-    assert(loopDurationC >= TIMELOOP_NANOSEC / 10);
-    assert(loopDurationD >= TIMELOOP_NANOSEC / 10);
+    /* calculate uncertainty in compression / decompression runs */
+    if (benchres.cSpeed) {
+        U64 const loopDurationC = (((U64)buf.srcSize * TIMELOOP_NANOSEC) / benchres.cSpeed);
+        uncertaintyConstantC = ((loopDurationC + (double)(2 * g_clockGranularity))/loopDurationC);
+    }
 
-    *resultPtr = benchres;
+    if (benchres.dSpeed) {
+        U64 const loopDurationD = (((U64)buf.srcSize * TIMELOOP_NANOSEC) / benchres.dSpeed);
+        uncertaintyConstantD = ((loopDurationD + (double)(2 * g_clockGranularity))/loopDurationD);
+    }
 
     /* optimistic assumption of benchres */
     {   BMK_benchResult_t resultMax = benchres;
@@ -1599,8 +1596,6 @@ static int allBench(BMK_benchResult_t* resultPtr,
         }
     }
 
-    *resultPtr = benchres;
-
     /* compare by resultScore when in infeas */
     /* compare by compareResultLT when in feas */
     if((!feas && (resultScore(benchres, buf.srcSize, target) > resultScore(*winnerResult, buf.srcSize, target))) ||
@@ -1623,7 +1618,10 @@ static int benchMemo(BMK_benchResult_t* resultPtr,
     static int bmcount = 0;
     int res;
 
-    if(memoTableGet(memoTableArray, cParams) >= INFEASIBLE_THRESHOLD || redundantParams(cParams, target, buf.maxBlockSize)) { return WORSE_RESULT; }
+    if ( memoTableGet(memoTableArray, cParams) >= INFEASIBLE_THRESHOLD
+      || redundantParams(cParams, target, buf.maxBlockSize) ) {
+        return WORSE_RESULT;
+    }
 
     res = allBench(resultPtr, buf, ctx, cParams, target, winnerResult, feas);
 

From 68bec4c5bbe2196f5fc97b34ab26efe9eff93cf1 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Thu, 4 Oct 2018 14:39:11 -0700
Subject: [PATCH 371/372] added graph for ZSTD_compress_usingCDict() in v1.3.6

---
 doc/images/cdict_v136.png | Bin 0 -> 33330 bytes
 1 file changed, 0 insertions(+), 0 deletions(-)
 create mode 100644 doc/images/cdict_v136.png

diff --git a/doc/images/cdict_v136.png b/doc/images/cdict_v136.png
new file mode 100644
index 0000000000000000000000000000000000000000..4a6d45620a06f653dd638f9511f7993b30cfdd9c
GIT binary patch
literal 33330
zcmeAS@N?(olHy`uVBq!ia0y~yU_8pe!1#cJiGhJ3`q0BL1_lPk;vjb?hIQv;UNSH+
zMrMXYltlRYSS9D@>LsS+C#C9DSRq^Ef{h#@;o;FbVsFl|
ztXj2dP1x$N)uBd;Mn*=H_PZ^z^fy#l*xe
zF*`duzsylpRs9mW_4Gy2#fuib(pTE>K8sOxL-vhLTNi(iDLSd@JzcML702--M_gQ7
zG*mbyb#`_h?~`3@ZTIa)vT^#kkgj`stG~yV-*ug}V~0hOg5$6I()oKnp4Q)gV8iyi
zWzrMN?^Qnk^YioJLx*0h{E$;oVKK2lL)`vhz>c7ky~l5F&HjE;eg1`=`~Q48UH|uW
z{L!OFQ#h6?aViyRE>$?Qa{0Wfe?OnAtEy_AKNbJq>A`_U<}WQbZbayJ>@9kFs`~xj
z>F0~L=iPmgQ=)DaR8{rsZux!RAOC*8_xJPL_w82p%iGgq%Wf`oZlCgg*QKW-d=I<*
z-pg8-y}6k_e{BZOxuo}ZcZ(nT>eoBR`@0)Y#EGi~EH6w-ETo>DF+SfSaD9FJ{@UN)
zbdRrBcJEsgy}d8`!lQqGe^0e}SsA;#Z14Mhzu$CMd_HS_-tzgJeYL;uRlQ#8_2)n%
z^TKoG_iM|Kc8N~-8ejjnb(TTS4TGZUr&Ght^6&k*{gL}$t5WBkU8UOm&z?UIexclN
z7x?LxR3X^?BRx7h2QK
z&s+N~n#+~B;$f?J-maI+Y^%Q=m{B~m-
z{`vKKy=U3itKq8+zco&etLof+-sW>p?UI&Sc}BadFAAO8cn%6Y3Czsj|F^8_#Pa!d
zuQrM7yA!d(R?~j|dxj3w8=>DJmvjSm5}Llf(E@_&cj3m-iPJ7hm-_D$#8?mxxxDP~$0oH#4*GG+15aH&D27d~vpM(XUsl*S}Gg7J6p7@JQQ|oUbN_miJ~e
z*`4S)d->8O8M(#DnkuJ1#O&RgeO*sus~-2?$n?2O?>I51hTQwGQ#pjWD9pQ1`oOIT
z6DKlm+~upuS;NKnOiOh3?)D`~PtFz0J8#@PyQ)ipH!a33?dy`l9as0)|39t2U&dSH
z@`i8Q4gMwp#r|sJ-Vd3NFSw<*{GKyE(TTI~@VzS=lik0ryTn=d
z_iMQ6F}sGkRcEXdnU~Hto#QW=Zg@Ov{?ywaSjA&DBp>fnTfvrBJdNdF_km!4+tv>P
z@qa@T?6{^*Ub*VmWXn6Pt(}SYt}5=^Xg{;@K*{Nxh-CSksb3!HU$wX~^TPgjyI#x8
zySbzA@#pjQ_xDzRkKCMg^Zu)gi`f@%J1$=@^Vueh>34LC=lU-~YR+@tTd8dOfAaYJ
zoz=f3wm&pXK4$aff^$Tmzy05m?VVFuBn0aI|1DRO+~AsUv^ebK%#H)cmhZUuVc$*v
zTSVqw%H#FaV!kmeBOmuTWR4b!^X*zCNY)VQ4%r{
zxy@(%?Axkqudc41dF1(Pk4F=g-GjvCQs%K2tdI(IoBjCn%8>Sre4kx;{%6)6E$yrd
z%=TJ%Y8B5nMxU8Bt@*;69gYOcOkS~|>fO%gpM)oC)PL;WRDUE@;-*w~S;DPp-wrW-
z`*4{5vUz~Z3-6GYTZ~Q4nqLH1lo>w+iMIuNFy`pR8u|L#5Bj_{hqu4>0|Np-4xBvgA`1co|V#EH0XS_dH?tIjx
zy&zilEW5C1RVHvZP47@#lp{9#@w8&(GigH>~zaBfH#z$G!SoA$%4Q(t`aAoGVUzd3o9M
zeEiqj2e$BBt9<3{zj)t{QwMi0InGe^bZYpjn_7%fyq&}JGFTDP-g~9G*LG41l1B=ewyk+l^Ix{)I
zN;>Mp=JR&HKOUE#+;HBK&tM0$QAzdxzuzmHcQ|yOOa8wjzxA%)I?twGf)7-exW3$7
zs`~qV{r}ZwmRYPlA6~6qf9dXPul2{Yy;yRV9^1iVVC?;GdR*1Y)4RP?wyv<4|YYkfaA{_liqAJ0@+HH4W>0
zF6*|x!Cm6u!>q+FlZ31u_l6cXZ{D$*$9U!qRr9A}Q7XGEzu7%i_c`+Vc!;!XFxMRy
z8yBBEu01{n1GD2?P17wn-u4J}{`~j*{X=)t;v@&oyPuo+?G*l;um3mM<|;dMeJxmWziL6a`&U(%=7QQ<3=7ZjYlI_9Yx-QWd%uu=nYd8ArRt7snVUANDs1j+pV@
zL_YtiUVWn2k699Sr8UbJDn6aI=)0j)P_R%#W#Pi$^{0iT%PlH&sgP)uZq+?bbNllI#=>v*>1bSlfa6dv*16=jG{PjmHfS6-_r(Te7%l^(<*Coy-}J0+L0JvrSEz`a@9JO`*;C
zZlL1sl9!9@R8m>@w(!ia`Q)j}Hr4lUnnQipnXIHM#cqFJoH*iO
zcrM;iO*Ak{v80VJ%kI?UJtsBpa*3~&+{N=ittX7T@z(v&FXbkAszOoj;amL~9xre;
zGG6C)Xy;t3(x7@R#*Rt99?i{?KfjziEhPEJo{7iq9q(9gr1ViS{PKsHioTcsL`$Do
z-c{Hc*czoVKj=&LS9Qic8v>KxCja<#slZ^p(8&`Fb8j(f|NdepI_;y~+*L2d<%9P9
zED1L0t}0r%esREuXU0B@(+#ZJ%3tK}vH5CqwQ2i{;yr#T9Z`Yx+qlK`p4?3~$eYh&
zEECUnG~;wbcV#+zw|ZybERW`YOYJ&dwR1JAtv)8;(A-?+Z~0W@x5pNbFCqpT`%_k|
zC}^)$aXHV;S$;UrYmZRCbL|aghaB%LQp∈_bKo#S4w^JrSF@=a|l#0}IdEG46A$
zd1YW>bgaBW>_vuA>6hI9Q=DgXSs(cjZ0e}7T_Sbyq|bAMgD*Ke$+!R8D|&UO#SXcX
zh7qgzwajFGJH1T1v(;QFP;--$ap?RYySFcTKJ1XXxrx!$;6&hAv8zG6!VjWPvwyni
zF2A)u#gLoVeR_b>OM@?Q?d)$a{|<@xvE}92gAE_&mfv%Hm~jMb5j<^_#=
zOD8$aeL1_Fc~wzC+JmXb_#RAHuC#g9ekq5iPo$RxG3QG?W)@>zr0(H#;q
zf0ldlyb;x=B|TA4m_8YP4vEb87cZp1gPfy
z-ZV?#SYG!(?(72(A|^CW->93qb88Q?f?G|s^^Q*)x6Id9VpO^qu___O@5a1aKXe40
z*FPvUjf`b<*!KMOu1S_hw4IWSLm$hYaytENn%7(}qbVgG)6Olkd+N^k@Uohjn8<>P
zmV~_G6HVq}k}XF64}08yxbD+rO%b-)8~7gwPv?v>6Y{K(R|wuNy!h0c#=oz-R4&Z_
z6lQSzlyrvu$#thV|1L~4oY;D5v0Kw5=de3U%Pg%X8p*H7Dobhod$4hVX6w}5E{A3}
zoH4PDw@{m|5NX3`z^lFaPmk!j{r{>e#A;=vo@Rf)mAzi`knz2iO)I~;%h$HNm6+AM
zNwRuXXm%PyHk;B#UIET%fx~fUbV_$#vz+2ORkq`#B4aX>!N&8xlKR?{8@-aOm@mJm
z$$#}~(YMM)Ru;AbsKbF`!GrLtm#whdV{H|Ls3>r!!SxzlJU7;Ej
zsQB?;q2J-zhr1i6J^4^&u{U7t33ERZ82qpB3*ga0!UtA*#=6!#+c!_`>p80kW$c=53HG?(E6^$*}n*
z)8+UbDHqf4tXh%h?b7Ks_1L$jS7%$5d`u_aKj`Jme`GaLI+UI%+e+W&#D5z$@ez|{=c%9sy
z`7IX(EBs$+F-Wz}vpH+$_415q@|E;k%k>hkR4iS7O;vQ3rgMUw(5WARwOtHPTDi`A
zh?LB^bw8^}VqHm{SVd=vx78^ouMm~zGE*eP;umZcJT~w20r~yh8lvkJN<#NdS@ZT_
z(ec;{mPdz|Hby;8xFEn@nqH(X|CQN6;;_bVvx7@o7D~V9*;(gu^bzN4!JB8gC)%BD
zDV^b2^T#~0;LOvUn4oWF*&e+KmHSkCCvjDHo@a5k^xQt
z)-h!*DP{=QRoXbm>i!|ghWCuK_*h;Qh%eTf#!ox6bV}DMR#8ly*i4H5;{WXREo(kFRoVVgbL+3`J
z=LYT3-|Qt_AHKby@pno%!*as|B^NqQUXv4BD)=_{o!8C>=Vh25+telPYAp!07G-3&
z-o1ClS~)Sh+!Emj{)ekOoDB2sSUhi+Ofjroma3Dld?}JyH8)?=
z(C4B+Ky^^;$r(M*rcJtG$=k&_iz&_V`Oc!}M~-}8+HgremM!*-9b@puuFR>^TZE=A
z?fS+!0iJ}LhmP%YIdn$#(axh4l1~FaYHjy#ICg8o)Etj<
z#}6b*a5qj8HGQgo$XWQ*CzaoS5(U<*$l`e{&Awmzv@rQp-?gZnZjmZu6O~mv{@@i?3~6
zInj2$k$T&2182teM3z--&MZsq4VH55O`0CQWRX(bhW?w%7j_uj(GNc?7@+gQ;mqrU
zE9MspxGC-Xu&25vu(DC~%fpyYPqfe4J^vzcY@W{l2C0g(9qBP=Gs#+S+Op
zXgH-+yT{~Z?wS9e@+?%Ig*whneB!atXZeB_BPk~*x#O%tr9!n_!VlBu*FO885Ywrw
z^4@rTSC!$r#a3QkJ$cLLmPzgWDD=s&p*bPq+8@pGN|x!emli*DH1u)!SlYP%3d^1Y
zd-%UpAN!Xf|L2KgMQ@Erwp6exhrdd_rmLj)C)uWR3lx`%yg7G7yL)lP;trRYP9M|1
zxp^|oo3dihD~sx;NzC(3u<6u%YY+*%D3B>Qt<*3{fnAE*TcbJY$?`st#*_SY3r#QX
zn={)5D~PY;K-L4vLl
zei&%15bC$>dNiYOQS6WIrp{!}X-QT*<+6Guo-_I`Nmd2~zf&@c{CGF%fMa>kWo8je
zjd&@qmfXM_FN5gHpW6IEfp=}qm6x}_&7B#?752nIdZK;4*so2O
z+)izlIC!zov2)vkwkXxlyt#Sk=sT-({$r%N)+PnM`CT&%QE^T0N)-wVVh3e9`k`b7DCjKOE234(jur+2vrFdBSV*!;oB
z!e{LUr4HBLJ%3U-iZAwyZtXbY@@b#r%{>P?o&<2MXA~Ax*?c{#vi?I__?f;X%enUK
zsst70u2D>Of8$n)&z|JF1+L;O@~LicleZZt7)wh+wEqn@a-{b=t(H?wpy;mP%FpGs>{cBrO#|p##g;hm;W|Q+*FY9IP&YX
z^{g%awl|bYtgALZn&oU*ARs01S9DeGi47Nr>kS1+B?e
z%U7zqDel-M^r6f!E#!*Tkq4%al|#alwtvhB<=}k2%ZJnVfPz+GKw(^tswxl1-lEe7
z3mzKDw|m&hU9invdug3BQ{ADR88VRt&Gc@o&iM@SOzp11UQ#A56GGG;
zy_p|$sD|m{zaK)EUwqiN`S*s;%DNJVzfbaK+O_Sp&Ch#>Z@m!Hd;iO0$%mEwCM_Kg
zI~r`~Tiu&{zf}3a4n|+GzOtDtS4BnnU$5#Hp6vaw);{Rj;+p-vO(9NKG^&eTCM>$}
zBWtl&vT%aIyH9~Kavl$g@gL9a`So7rtFnoQm9f&zA5HTF!}si45YBFL0v&%5Z@
zv{hV8O$}%=hn72Dxe*lV=pgVxgXz$6xz=zaaTb|@o!$Sap|h2289QKEG_=F
zTTe>DJ#@@pkL=HeTtnv!>T$3J$Wq-}Bk;wW5wH$O9V`1R9=y
zyn5wI#?~mQ3Bf1)Z9X1hm#a{SkP3dK$-%+2w#$Lx)pD+xO{@HtZ@!tcmTSET#1jq-
zpE3ou&I9|RhLiDOaC{c;L{|X;mOqk>4$p#1AG|-$tFnoksj1<i$$Weae(A
z9_1gUx22DMpLxK_(&E3DVUvg=C@vZsK4b=%Yz_$$
z5MbHU(C}fUxWhCh7Dh(}4FLf=-mbK1UZ8{}E+Am{mFYxK3n6N?44H4}5oe!32s`sc9116blbmo&ELp>22+`Q!m{E
z&yQ7zFgCjv=ifc!o9nZG``_AY?;_ir+rIt!R<-tD$3u^;JFO1SxOn@DwX%xxEs?e6
zuO6Q%9V(7z(tCX#^%ie8#b20pt
zw(iu^Uej||+$cCv;E{ZB%l2*W1#TT&w)Lsx=SOEh&OI~N$am(+V^0i%XIxC^x#E6I
zB58}%+WqF+PUwSky~2c_TP}q~E?&vF&!9FUA|qn<1+%}4f8SmlnWVjP%{rZRdfP&8
zf6}Rp*4p-F%Nv`R%8K-t=k{LPTX}6y?yOwbS&Ld`wd^~#FZXQjYM#)r&0Bjn`mz_d
zmmMoBTxPNPUd{z+r!9)#ZhA$4;#e4zAJ&F-pNjO~?e;=R$w%oKve_OvkV*k<<@50nVuhg!<>@%y+oi<9#6u&8G6I=Z+pa$|(W$$!b5`-&&o%+2
z4+5?(ymRW#tW~q-oSL&&wAy@H&Yh&2LVy6#roMqBU6p$pe~X!vI=&lX?AzcFjgQN^QCi?=OasO)x|FH!bU
z=||HxsmYBeH@yf*&gk(|J0&AH?S4~3zLG-Ctxs>8lo^@-hxlf#(kyLBYpG=v>s`8M
z*Pd$|ve#;EpY}YWXw#|9d^fVb#!R`xyp(;LSYGO@9QU(*$+MCt&N}gB)|YS3z6t*7
z@QO{Hxl`xpj7b?jZ_?gOOYrJ2aLFi*+I+C5^u`LqThkxOm}T%(_1?3LOJZSkT)-+Q
zV3+G?a5tUx<3oc5%e=2I;qQ@M@h^la)HKxfs;ld@MMmB;*Pbd{^Go8t+0=Zk%~N&t
zW8~rl_jzqTm9_cZ+I!V!3Vc>ySG%tAo%iMJBo9u*wi`WW%{M13@y|-no1Ws)WqNGZ
z7quUzzAv**t(F@4B?+TPbTj@fk9-)VoBPP?Rh)jdZn
zIyF@F9*4D8S&r82ExW6>Z#*?4$&>SH`y!6#SDrj+eZDkg`I6QpU-{qOzjninrJiN+
zwG<=6WU=gyr3(aNZBrZoz|V4=C14B
z?bX#ImfX58DgNj_*--1U$a2{Yavj^;j)xyTF8!$Rr`8Is6+Ek5_WBwxDZNsf@{MPe
zsq|T^X1;7@xgCMWR(Qmw$hvxNQBdHxqNGqGwc#&A{@O1WO8x~NmJ04Qb6UMlZ{50;
zZly;v0+Ls2uF=!kCi-6IerH?tk)YLYU+L`f{W;rfmG`!g+>_T*#jlDiy|QFU=H*4D
z7oS9~T)V~~+(kn9){EOE+;7=rk4t9#Etu)teZ_57&oYj+mw93jBv;1po`0(=n
z1CA^5J41I)oaV9lXZ^noY-<_khRj^hW#%6Idfr6axv$SvwRgShQi`5zD;@?85;Tf)VI*kaMv)GW0{Y0q@q`D{j%WM0@EW=0kIb>w`@L`
z^dj}$)EqB9^OobUlGa*j%bfnz5mBon)|TOHqOBb9tXo3oY{tY{6Qg~7-%hfO-#KAv
zYSy(ET!#8m%S69FcY7T6pZU=8uaA|>?B-bxMn9&w3Qv7FwdJc-t$48b
zT32h0ECDTpx@5&!0(0Zq*Y%#`7JU7+D(%LOrNL{Z%O)+9Rc})$*H_KOxeY@bus;yzKSFgV5AG~;LSmb$syLzurBC7uN
z|Fvsgz1MwV{;B!0?0lD%MPmtuL;`dpldwtvWt^Jo=ufE7Cc%XYhB&0obvE%aA
zWsd$IZ~frOZQJLve`AjF#0%z2A29JORFLyY)ZYA29kafS!X2`E<0asU*op=
z-WvTaVxK}^B)k&H;z(7O{dVo8)HTtl_J0p1veYuYlYHy@He0;PA)C$jeaAhwxf7Qb
z>IAe^oslYDzQM2R#YYQW#~n2Z5!Z4qNG2#L8MaB}w7q$iqam8rc1R&!?uqcCl`B#`
zy)@VBgd6Cr);ui5%$YXzN*JSlvc{%uS+k5@Y&{jydRM0G*k#FriW!9tlk`?bg?mqA
zSgL#Vl~d`p4Y!J>@?@{vvuwkGAcl*d&w3cCJ$#>YF>1Ie|?I7s{Es=ZLNyyopKfZc5hwAS<71`92GRh
zuFFlNX$h`Wm^s_wvz$ZF|1`d90q&-qf?}xQAq1^OOrO4o&@fqV(i*
zy?x0U?&Y3RwqsQ$akkKcH=^Y=w
zf9d|K7guO%>vMBw*9vP*vEBXnO?QxfRmtzlUw^&s*Nsd%dpcY2
zhl(($e_+Bi^Wn_-JpZ1ooab-5^ZCik;dA!cucqQBG25c--@c!l
zx1aMfUv7Wr@xwD0eDtT>UZKP^(?2`5|NnVW$pD|k#KPobAJ#u!6Pfz^+f%)`nkU=u
zf7}0W*G}*0FY@jR9q;3PeS7-VuHGhpAyA|Hx*RX^?J*;{PX*tT=q+TT5@kw>P_!yIn@ssSNn;J315D8$y=C>$6#xI{`|N@pdjGp
z*kE`_o!KsO*7h&Qe9iZ5dA{^{mA$!1R>o>>u{Slp{!2S7@3#Kv@-Oq6703Zb|9?o8RY4U%NL!
za%N@z^XlDhWx1{)E&j0$@;4{Hom#zZ|DR8r-n?1#e*gK~I|?ta-7b}!ym)tczpSYi
zTed^UTQ%7tPNt>?eT4$w-`VGKi{BORo^M_M_UGlG<#%sAymrlK71#W0*X0jQntWJq
zheE5YbaBzA^0%jczS(?NfB%}w9}h45`zzee(0;LMeo_trOl$ZR&at}e5`?V41EmwDcq$jxc@>;Gq0J{4{D
zn_ly$-mRyX2;uu>E*V+OV%%UKic;xB2&*u
z%&wHt>u=!0y;UFe|1RoYRaEq8n*RB}ACLRy@6YAs??1S(^w*aka<&The-%&I4T=j(
zmSaczrOj*Z>}meIE~5VEgB1t)%WEx^bq*IFz4=_HCad4=@u6icEmJNRwprheii_je
z`0wNKed6(VA|oVj?kukP|1a?IarY&UC#p_YP~h0Y#3a6TZ${CjC(p}l&HtW%xmd`3
zq(XPwnk;_X(_Ut4l6TbN*~+wSU)KbN{@B8J_Rn?7;drUa=_xI;{x2D+E|C^Y4+QcpSY#Z;h7cT_D5=-AorJ1yC$(6pe(^x9+@aJh$
zKx;i&Sa!LVRd2t)+g|SP$A_`}=j{#tfAibk?jQ+=}9>sM(-@3pEw+O=wukaya$b-S!otZz@5Cuf-E6XDioX(KbwbbIFI
zDW9H5IywlfkZ#=Qnf&wn$A|sv?-$fQpKFw>|$B*2wDtp4QrC_z`lI-gj6rIm#
zYChXw5L~iF&uZ6Gl}U408WUU|yfI7n&oX{F^X++`Sj*C^OIKza=RMQN=i8uldE1$1
zs}3z+dwI2m#icX*7dkKQ|DAA^D{*6psCfAFX<^TvOiQ$xMdfy&nJB|F=c^U3K%@7hxE
z_}Jmu=GSY!Oe}jHCgop$=j+uoZOomW3uAZBvM#-}^G@}KZ&TuDa($k;P0TS$y79}0
zFFR|W>79+O`oHbBc#3Yt?&G)TDp{MxYdpOCEcwMvnJ=E65C1Hl;Mi;uy!_w#4W~|R
zTC?WVn>SsjPEF#K?$eEGG0rJ{e(ueaq#N6K^LXc57_O=RFR*r+XpG`qtJY)3XWM8m
zd!onjL!N2zWB30yf1Y3ezNc=*n*jCR4=ev~P2l#-F?-y6bpNe4ewMDqr*;3_j|xsc
zc}nZ{mX%*#1g<_gWply8)SsVL&5}yKzwg?stYA-14R#SRv9@j5%R76wwzatMNgNQ{
z`0GVRZO)o&X~}D+$F06P^Q=|fwJQ=%P95TFx3x<7Z2B0>qL*J?8R;7NcH#E3a)&zW
zUhO`6C*ov+bFG`3hnw4+PfuFKZ=E{5y8r*_xXs53o^%AR*0rzLa4c)h`W%ySoG%H*=jm#mTd^gHVPFLTO{Q^
z$~+x&X=*f^yZrg3TISbfL{klBzn&B>5HLbp+;wSJSkmXf%Ch4FZ1sH`$9rVp1!6oZr4IWuL`Y;TmSloptxbNTbriZ
zv5d_#g@ku<3z|PwaNVtZ^v?=L*O2`V2YBZ{d;0Wl_4~aVQt9(+zwIb|oOXWR-;3_@
zsVOOE=2(6{rM>=3w^Y{zyAB4w*aD^hCS~&yyXF*Vq_L}A$Z>TQjf@Os=eu$#&)2v0
zXwbf#$d~8l?uy-QF=v^TW#sjJd+m&Z!h?i@0vnCG*GELmx3ik&mvAiANY`f1Hvxmo
z_y0(3k+4-#GBEm~pr#bWpzvT{^4oiRf6vNZCuv>wMtl99Pj5D#|M%fA|K95Fbzd&J
z|Nrs0U)aw=kwIaC)(;V;h-Ay;ZQo`@mc%W8yDM8u%iw*~hHpmZVL?HqF9M#fiw(WK
zEwZcY)V5sfZTa&}t#7kEfA%Ef(W5W6)l#>%8qe}M_NmBtS__{n&&!urJ3B>lg!4?Y
zj?8!v%quT&Xvf3LPft%jKg%@x)02}S-+n%yKi{HI>4?{lL)`jzHYT@!$aMI^Sig$p
z9J57{WKz;U^^*7Z_%7W(bxMboHLQno>Qsvfq0gU-=iW-Wxp()Q1C6Incx3zZY+d71
z{=lK*>Z;e*^{3Q7e7JDq#)T0=MnRV4yvS($S&ava+nszF;>+^e1G>)X6*`(%FDig
zC-CfvXoM1kHW%;kSRY~gq?Pmr{6j&B7NGe
z`-W=GwpmIM*@_R^{H^8o&*-!&d2vBHf6v8jxwk&_6$h_X-7-a+!-SVHOlHHe
z#;u8M`~5vP-Y%EWW0AF9b@SQ=zC~-+tlG1u==-}*TeGz{ZT|fI&B@rUr?zDV_Z|(p
zyt~}P%gZA)G$Sd=W4_%RmbR(WX6mUOydt9_(896fVX#1=5tp6mY{_3+(r4Hw9qADE
zw-KDa^2D(p$5|9N1*OWnE=XLLeCtAh@_&zGh>N>Uh
z+my{YI{NE#{{Bkd|0gIiu1~yi+ajeW4;)Pku?d?B5Ki{wWojb<|wE5s*P5wuniQWPNEU(-g5YL6323H@A}V;agj~Wv#a*mzD;G
zh5b9!dh_&j`G^e+&l`6%G7G!)9r?cCVWpSXe5V7Z^R<)~O;|Li$Z+=ls;{s7?f-t+
zeBRF6=Si^D(wA^Rov?)(?R9=|GF}Up;lQ
zV%h3bXV1oN$+&oBW$^Fs@7F*3aL)RD&fQ&Gmn>PbVZ(<$>vu11ZPnf$;ck$BZ_k%6
zU)HSAnX~=h$Nu^iFJsx<`sI4d#iMz5?2!(cXlZHrWBbD|Vj)}0*6GCd>DDDBCCU8p
zpBucbs%Pt_O&LeK=7g^1ny#~QQ$5D#;m+3KUo
zEvF;-TyWeL7Wake!2Y6t|*a@E33P!t7v9&va(^OrA?FHs$)9Q+uod*sBCdG
z>7R4H?f*ZY7e;O4Z7H6~8$5X@&xKf#S!r>n=dF)b<9p!obSoo^mG6Yd4HqZ;I~HH?
zAzULXGhlUZzrTdBTH%=)T4xq`PZwJgzyIm{|0%~K&a62T-&+1sXxcQl>CdhvRkbWm
zm9W3{y})RJJp*gLN#gJ5gJR2m=`4PpHL+{=v&+WkERtVdTH3SY`r7FBjnj0OEx7*i
zxW3u~i3a0i>0SF>RaU4A&-!zaU4BW1#GI6#kDT>iuZHXI`=J#3@J{jh(%og=mFY8@
zIG*`2b0|1uGfm$=$-{QHkagb6_I_tNOtgU!NH!PdeW3zw6noN4(Nr
zb^j`#-`P1wQuxyP_=@}cHZGqhb?6`SoUrJ)bt~6AdBEIf#$d33fu)KycG1!mOAfw$
z5Y5@xbW$T`MpTPZ%m>Ztj~h+pZ@=f4u_$93ESGRoLIg@xi~}@1NHeoPNpW#>vTfE^33j>i3mr
z0=*aV^O%;*QaH|dR^+jen+gM`jK+*z8*bkCVJgObk)zBx}l*ZJ=3I8e}%N?PcdjXu)1*`-+Yt#L9riRn%}Qs
zPS?EVDKUAjzy7WlN^G}QO*!1gyZgx`?-{b6H0FbL?)7~9xq8a)FPHrvzdIbL{b$en
zeZQ-|zG{^h-N*QK4$lfEmXw%Pe;g$mtVINxWY|+3n3rm0u%!4Wh>8XOe`tRH&b-_Q
z-`}5)+^qJy*SxRhM`FiC3De4=*~0!k2|?kjpXOd_YyG+J{tWS398#^?QA-ru?OcDZ
zU$f@Tg@w(ts;BG4e(2WH(|fjho!g@`_k;46J=(bbPw40W-)`rBG&s6;>Wr<~*NxTFGN+jDcuDq~xUBd*)BaP}nrdFH*n08yj)L$_>lr==Mnu0!mM}MsH+MU3
zSMfnXJ5us=@(NWSo~1QEpH4qx@$gc=#p8Y1b}1c4+<-&FU~NhxHjp3anK6*iK2Qr`Au)
zUcXoDF>_+7;o;KjvDZIrQdz2?z|p{v6>z0P@XzHl^Zp+1k<^*}^5xIYrw<(OwLWaf
z{4&|kD)T@C;vbt=q=FZ1U
zpNO^2?vqb>W)^A7DdoK^rBzUN(Ry|NovIuhOuUW*9I+~mNZw!vPk9jL=ArDnpK?h4VD~JQVp5+Yx6YlFw$m@1*;x6%rYu`
zwDG*%?W*Ty$I3As51;kw
z&|%~Edx}5DmM=|k{dy+u*uB+(+0&oA&~LTT4qH?3`g*6a{pnZH(G{BU0s<^i42<74
z`dPhPVmdwc8dD`}xtQLMDW&f0d@?KK#kOe%#yGzAS1Sjt?9dl@VVLgsFup#u{?EpE
zkJp=>topu79^IU6{<~|_g2|nl(~B1GeYE9C+1ov*(^WxBPHF@g(`HuN{eH7;_xnD1
z>uUniubYK_wR-f`bX}0qTa=^M8g9?GT*oa*D`4*Z^n?1=UpDCO-z0KNUrW=8i>aw0
znq`uo?biqH^10P76g#fwCV4&(Vid3W_3d`Pe%u}jwWH~Bx>J)w=ghBpHZ%Q5{3%JX
zPC=GGQjPQaZMP*oIx;7Jf9mu4{p&okIo~(=N5)oNi%d7X5x=wOX>|VHR(4VK*?e*~
zGw$9C(Y!h9NFwX1DsRvZ=LgFB3drDW&4_jDzwCv#WE{6@TR)Cw1
z3fG>Vo?a}XwENNKyCR$xXIGX!opuQnM8X_fY~SzMBVQj=e>3&b!}}~1ub%{)>Zj^Y
zTC_;1BXs7q^zh~H?(W`fxpI~p$Swf`j*{Qs_t#4rDEz-3pFDfJqx%7dOCP#+e3)MJ
z)8KbS@jT-!&Evfru9}E7ym^x|T@xIvf20_bYixppq;G0&<>Qy%zizI3YreqzqCcFI
zLu{+R`S|;%ui|1475lOI?i6>`$)Lqo3$_Y)Sbjb;;oIBocZ&V9IUg*4dd>Xaao;Y+
zf0O;~I)95q_ibAD|Ko9a`>$7mBPYLFzEyN1E2HCrMvgDD&ENM2E-u>l&+6foVCf$n
zD#d4)=S%V(xb~cIt5xK`mboI?n#R93r=Jfy5%TpFs5fkRf6LOA2{kN?
zjtAHUGpv){T+*H`i_ZV*c(W#0Mf}&x`v1S1*M$rG5CUGrVtvH0(lSicR^
zcG>RxG9}ng(xM>Ya@UmQAkXa=c=7ek=5{eXogZ(_{|nvzeaY&R=I^hCJEt}W+DrTH
z^i|^ote=ml
z{$r=icbq%4oVU$Oh>Pi9uS37P{MooID>BZ`deF9T-!1O>*V_B;2(z#-iZT`-@4NeM
zSN7e|@T)5Vg??Bw9a>(0+_{dEsmb9k$CjCvmuJk_@q42CD?gztK2-uL^?{{N%y
zUCXq-?ChF}i_Z(63zDz>a&e-vdj!kUH9d#)yH+n+#}=+21eh^Zot(`-|QA%O<%g_I7tC``0P4
z^GFz^oDkSHYujn}CQx+6aBSKAWYUDPw`G4`#~%!RTb2F##PSvM=WJbWw<-O+TxzqO
zsT-$@yZiHJ&#YWdftEUQXtDg-_v=-o`F)+3y;du-V*OKPe_m14tkQ^fPqr`Z`2S?G
ze?W+=($nP%3LGq=Op2PClAF`>cfPmtEiDZQ-LvkL)h&P7OBcD(G*17y#Pg)E+KHI`)po20XXo$x`D*q0uq#1-
zR-~lePhCIB!%@LPkmKbI!^hulU%y-N*wfM8{q3k
z$0k#Bqqni7U!NESN(9DCFQ>-{&6~CB?Y-)``K+Nm)pkes&KC`?cyG*j^uYZ6GFOkQ
zh}!*kY@HGk1{xl4T=2a3yllnuxsQ0IUwI0D*na23%TrUe3muPMEjif4TJfXUNxtL3
z#9Ldluit<2p}FC`Y>=blf`c40Zf|>Aa$HtAZ_mbq{wxVj<~6%=Z=3Z!UwrS!PjPYa
z)w`D#NXXkW3-GU;q@ciIBHG9o9-G?Jaii~d$cDJQA5&JUv;Cf+IWOpkpQ?%q!<4I8
zfB$?wZ){}LBxdmNt&O1MvWBCj^u>h2pHFA!?fZEozMj{K=gtSs4x7JUE^ohA
z^*VR^-LSv)-U=T)RE;)betYH4uX?re
z)2C1N|NnfxoxlI=sZ&)CTg7jbYaM8q@Cvjj(_8UE`MEi={&tb?w)pK``sw@48mS}8
zD=RB4KEJ=UwYzrtjf|{^hg$8T-ljq{rI`+lvO&N}rGsL<;dNSH5@9eHzJ`n-?IGiU3slD11d3#xu9|Ns5|
z{?5+gV?B~@E*|&ze@r?*W%bMXb-z~rJnGE{3WXIMUn-wYwf+2T>YtzK3&Jk!jLZ8`
z^6}BpHHN89mQM|ligMn!rs0vo#uuyIg4NW%7uYbV!JepDRahv~5wx@^r?Q7m{y)N-%
zbCJA%-HWeh&F}yDa@pT9P)k8@q8F%;g((G|EoGG8g3wb^q0n&m4D{gRBn2nGHDKI732hg7bT~4!@5;p
zuUyWwGDz?BBRBidd5)rgr_V`xA!Oev!ot$XreN@(*L<4qtSKgyMLX^m&gZQ-cYcyd
zhyP09dx7uw|F`>gSy0(+PW8K;uZ~Qa9~yo{l!c}7nuCIlvGL8m?;
z*7x(xiO>5{a(S7rjKJQkt6F}&$GUDU^Od&w`TXo`^RzQFT)V|s`FMhaLCsqmripv2
zw$3+9zABksv2VpS&4ax2i#)EZ2wc1+YHf2#PW!1z!R;PDK?{_d9R6~Y?EC%h)S*MQ
z`THX4jqLZnZu4j7lQ}Un@SFFNDOH>1%-3By)xa(EX^f_QtHTj778b^QC4&cd%WukV
zt+}q3FZOEgraNceA3u83aam47llT5j7N;^y^6u;qyUGk
z4mEc@UfeHdd#bTkQBjfaC%C8=V0q=V;B|8UU#pxO_rl|S_kTLI>(Y(;Wef{b;e6m&{At4sh5wE@UKZ4gZH#nSma4_@v-0kmP
zt!`(#P$}Q{Iq=kixZHe`fKqu)8NZv$_~S#qt`nMJQJ7Ryv&VVKAJEEe#|3pPR`x#@
zwC;Sva7+DYYu%|xwKwOE7wlWLD(k`m$2C1nB5YsF?^UMz`SCH=Uvmf5+d1Z;_7O$l
z0-!bJzXUz@KI&S0zjnGsn$O>pf1a*eT-VhqvM8me|GB2!ADz}kYBCvC#m{`6{o4Qg
z-R}H-KbK`+U$@5U=^Nqtl}k>5B6?q^gVX4@DIS-&x8=LP?d^;JKgPCsw?
zTt-!Y*}@w!tAdyNWnW(xn)K_-%gNKGnU%l0Q~iGL_Ki*M#|`@ULT2-HF8R~C>H
zp8Wr~yq#VO^U>lb!D|jjc)XSW_wV=npOz^n1b%xwPx)4^s>1;)UYP|wY##R<%DpW$
zC|1
zWik(K-uJCTdA^&gYp=Zhzb0)&Lr`|A1p`n01W?H<3QQeEUg
z*~OaWkg@UEySqgH-T%^a|75=YhVGvl7F+i0krCF)_KXbt^pp$4zSbtIF9s49i4fJZ{oinmrdsfP5yRV|F_n`JrYvAx@Zj9F&wC{_Od+E=!AB{?UOVno;0|<&42WK~Fjaf6g}h
zS^c%u|LT*{qCda9-I+yo
z``7K{04<)kYjt?D!0~4O-lxGzreQPKK59xY-1J$qeO5t{Nd3*iSIzu(H=g{TT-eL%
zeF;?CJ{Rbax4N?Edi1AHHsLMHi`QNLAEGPQ2=XSr}w>$UtHc)#sN~_{!iB-hZqs#7O
zsnvYi0S?q}94b9>v0JLkC)G}gZ@v5B<)x+G`Fp=!`+Uy&{K6pbX*z~dy?hft1%tEJ
zGX(+rM5k-a{1NN-$K{)utIxkyb*=aBl*itNGpx9;Z_kgnw1;$dn3SfHh^{9CKiJFk&U%fWk92`vNI9@0#A68N>7Sk`=
z_f*_4@7V7%)_=cVkC(6iWBA--^FopKc}e!I5rxwZi98TD6;TAWp8Ewl-dtaw@!Zc|
z=tf+Kc}>^+m(lrqOTXPrH(q-_ZiNl!{6lLetXZ_;)^pWFXc#$Ml1$Ip_bn>hug^|#
z;mpX&|Apo?r?&l>BLZq%S$iC-kdM8p7*q$!iV7EeSPlgTmj(v1Mn-;QJh}Y5&1W8l
zn>l9Nrt4{IC#V0raN)wr;N?P>eI7`CvhdpI?aow^a%Y>=q~;}pEG&%nP7cyy$MjbG
zjIR;2uh<~pxwhj~njP1#x7quQ_L|KIQH_kO!o_xG1>#D)ip
z`|Tb{?6WL>7E^jPRBo*R3W>3>
zG&VRY{CYCkn_q5?Rmq9>Sx?&hEhq8wrE1C7PB8qp#B=hzx?h=W{|pTcbtZla$o+Vv
z$vM92rRtuohYlV3pnV|STQ~dSB3J9OHy!=`;mInX(xtJ*AtEa)!==Uko$1py|HL4_
z`-jtBU5Hxj)_ZGL>1yW`@4qQM_m}z3){EWs=H_N~P0h@=x3)UB^Mx8c7kT~WO-|p&
zRqNK}t$d^+4=QX-S(29fo&Ec#IDYb}wKkvz@$yTTf}Y3BnDOaCi9jB+{htraTrbt<
z*L-@peEy=P>!P-L2{Q2OCV_^S(^gD5%QA;`vczA5h1;RSm@O-jm=+fJ3a1cF<(~D-oI8V>cn>pVO>LqgUU+0yHSbAMPdcWNk
ze|he`*OTwLlyF+K?~gf~pKV@qO7dcOOySWdCnra)wNE(EFk!-kudlEF|M&ZS`Mt{J
zwujB)`F_i+xVONud6%5MW!aAp4>zb^4xS!gXZi8}kK^_pof0wM`mczF&Dr(Bc8`7O
zQqCV0ug+yVmkY|g;BPo^RN%NZU;VzDyG7nT(6{*UVD-YSH+d#b4CLf=oo(j&_2uHk
z*Q)&=SAI%eYP_X`dzrM|Qu8@guU7ha8uA`>(p=Ho`R{sMHSgJr&a$j*{{H^k|EDE6
z*;agbpuK)iP(Z+eKN7)-x3}ds`_HfWbW+kdZNv10*hvfm`F!Pv4C>k(
z0%EE@v2ikanZ6XNFfrco(NXAQk@JVHg$oicd@kC_tFQO+&*#t6bPWCH=kal8{|NT=
z@J!EGdDs2!XU$K%ayAm{HQmmZ9Dng5!!=3buJBwTwv1h3%@kE>LzTMybs
zzW91eOUuj4%QwqhD;DrR2U_aZ#nJNp#lqa%+fvTX3Y)TUqwDSm!Qc9%?CbtKdK{f*
zxbN6=kzmc>AOk7!X})cLK1eOOcH@xLjD7y|?f(C|zTaTRqc2v~-*U?MCCqYWeA3yp
zalh#)&F7J^<#$WZ^q3m>d#Z78Fx9m)s9m{Z|No2f#@L?kikddl)codfOx_ePZ~R8t
zy|3kv$aMc>KVBS@SQzEyT*fJ;rFExwZpphlJKtokUFN9Rnl&}i&Pd}~;Ma95pwXcT
z0v;B>-#EYD``q2M=-~1d^LNy2@|&M^7%cuRxLT*HyBjp-^Yiob&5b4>b?j3YYtC7EZEduA
z@#}96o}fD3rqRKtsOa1x*Lf#y3RmuW7+i97cHXW|(JvaSB!c|yem>cld_1n=VXMFG
zSCQ`5wu%#;fEsj74P6djrf5FZskGmBA+Yz$&4%ue`}VFec=>kw{dH@+cD-2CUG@Fl
z-xrJfCo$Z9+9j&}CcW#;{7H}|lWvoP&+~J$nAy9oukU|zd%LUAmkhPMFTsD7id;%5
zlFv)M6+5xbo^s>C%udZzevvY}qspS{0Zci*kA9nq
zaDwW{^@1F^_v5C|K0U#{@J)pL`VMKkqbEh~rTE2L7PjaHMg+{CHEWerQ0<;7rFo#i
z9tNh5mzRmIi+cLI=2hCB^K7+ip6-9M>9oYx&4T*g%WrIbzk20g
zf0!80^_~0uEBn6qRo?k>)@41df8NY%=bYN>al=gh_`k}}&r(xT3iRf^5BFCA^(Gk{
z6sit1oSHD<)ZgE=d-U>tl$@TXn|!P%aN3#8k438OG`}WRaalL?U-vyC%)-)mz~O*^
z^yz~-JMI6!d3?3-@baAIrn6J5-rU_io!4`@&&)+v*yp8Mhsdv=-J#CG!Nl0?@JKRU
zAWAmUa!A=>C%lO{qy9@_x*YmxjC)&(@FIetM|ni
zPV@!o;$mEEsAOn(fBW)-cRy=Bu08j7_v1cm3$%&X)*QbB8R
zRRuaE&2;|%*jS`Hai;sHg2T&i-n?1izW499+q=u(r|n&7bS&LCvoAX>KBV76WhS`c
z@_?&x#$>-uIk!SeA2h66^Obn>G3Vywbe-_V)JhbMjw1y*dOy
z10pOOE-5K9f`UJ9%lUY7*RF@ba~4I2E%oRZ&#I`{A(VD?Dp>dpn-}7iVLo`
zeSZ`A<>_?0XnpgVQ*5>K>;L_<{eGwT!2!mQkU2{~eg14YFaP18)-|&~-Hhr?0NF5s
z;|RY*z=OkTA?7t+lM`%GubfX@EFNF;(Nlfyk-io4>wamro4LEWv86P7E(HzmqziUP
z7%lm9Vd1H_w`+r)%zt|;mUwxc%HRL@+x-83+JDw=bKm*plK1*uuU6f-5pjP_KX|mB
zv8TaOPfyF(xcu|8D|)dt8)m)xIc4VG3l}cD?u)mJU=;KObp^N@e|&u0e`fxCtK3=q
z*KVDym~ZTNmkG>*h+^Jq>tq+0XiIr~15t46|$V
z@@{Q832P@l$Yi;(`+S0#xPJV*mrb)K?3cECdSJ!H#qN*yu74x9c3$SGZu
zL=f5xWl+1aXKnQCnVsr#<;nJI*R0Xe(@Q&O$Gy~|;e3XP)5@^bQ~&-F(}{S%*{A{<
zuoYl=<*x8=&1Syy^Y4GW@nud8i;-fTX9Birk*CZq?k
zr>$XUvb)$ku|GeOPgOpzTzjESC+$AuS%&pCYior#ePLr4=J!`
zqGTFZ=*4oGZ6W+!p
zo6l+8+_`z#s#Tf&)h{%^*K}ovOxKIOSN(o(+}^6fxG%rHzTUX>+`21+1Zhi7_~7
z(a!Mock=gtb$RA{?(*51EBgEY7!^D?a65nh-F*yY)3h`-53+*VpWi-$TEvWw2eLVi
zELr+=ZPe9!*-uRuD474!Z#amJvDuiAS3*hmfjWdiwh1
zf2;bZU0QMGZqN5Xo6VaxC0xDiZ!f!E^Ue=mw!^bbvu{LH{r~qj=kED;!9PL@J>nO8
zO+5nknTdGgg$p}0gNye6x2yZ{;aS>Mr}MiX248bG{C#D2d45Pp_t8Z^J{&)CWX9QN
zpcMg6p10Wj*wWfEF^2^d&lxOC7w1~Nt^In{&E~s^_p(i)l=
zB=$-8LC8M!`8AXFuUxrOPAqhjIH+~-AfLm9UpmaPe4VQG?8>qkes>;HZYSDh!k^u~ev@KDWUd8D^XBmbt6|MW`-{@m^Pevt5NH>02`T3dY1|Dz=sHve{pyU0;9^aqB6G
z`85keeu$WBs{N9V>sbOGWI7Hwss{CHc?yAg<_Z@+aY*naCMN#e|L+xh%_)|U^Yd&ACor-4*#>U3
zcs{3Cq%^d5BILm9wM`C>W~M*7`ktHH|89hgTaUy+HjUfca*G>YuHXOfkSV;E`Jk3X
z>Gya2*iB15y$F0S{x)FI?2mRU)6dO0`7==GiNA+OLam6D_Qae=-w$!?TRc$(bq5^;
zKG-n@-re=mx?Jz>rlWJFJo$U+t#`@!<>hxv#kqZym6fN?)c7%_HAv=sy3x!x#?ZFR
z+E#~@*=9TM*NU5Gg=BLsnDV;Kf4Y8rU(~5cgkT242}
z9sh#fd4k#zIXogX>V`u;%U8&^_7)7;SKsO4i%r7L7zUK-}dSi*l<26lNZ0=?=PR$
zQ@iZr;-bFk5|HXbuyMw_yV|EtY4OWsh)Uk+`M$g4WmClSx#e=ZzirLGFW1kqZ27&~
z@3CTDbBj)Cu2tG7q6KPIJH&EGBq!h8s{g%R{#I;VWo)L6`1>}0&5Cn#EY0uNe4d#;
zkF#B?s7KoT-Q4nfALsZ?+6)dbUpIwq`S+V-t)o(ZyqV{}^6KXEcHB!hWy}BeOixSW
zld;%vGIT0CI7&WfFcnT#limH8FFJ9dp4F|ztd{BTPAp$jrKO>f;Hv3d9kbY?r2p{Z
zA14#$cTfHOZTC0JbahbRDsXJkW16_;`IVPlWpBGA4Gx^1WbsM!w!7`WACI-e)-?F<
zT(o%cO@7yni;9k&oSnb#=3@8$PGR-3cXw`{Zt^OGl6r(X6*8XL^8@;Lk~Xin9kq|5K_?!I4kJ9n;CsnB#y_Kqi@soft;
zOoofyQulrhySg!XnalcJ->hFP_j$B=+Lt@U=hw1}WR%?6k{Mt7HPklq$uw|qu5ERQ
zh?;dVn-^C$u_SuV$jya~;do-Js=Lebj1?C(*FIU^z+Inu$1I?o%_Os_}`K>(1
zsp1AMR^lBL@_v#c5GO@{ZyHu{L{6Imdb$hpG~(z$o4$Z>}wUdyXzhXpEq0n1k`I?+u;zg
z+;3Uj-cMoczVz`NT7K_fq?aIQ9?wxhO;Pc9G`D}NfAsR+r6Hh^IX-e2;AJe``Lqolh#J>4cG%e8EC;(0fVDQTILow-FtO4Zh`~+l%ET;
zI9!%4f36epfR(cae!|e&PKQG`Hm2q{XKk#A;&>SBudMTvo2kh`TUp@R*41|A=Fh+1
zx7BU`uyVQJv_GtjjtjmDcGy3ia;WsR4VtK=-JTDh&)e#@zyG9pn|arr22*?c`EN!RT3;bo}Ou|w@rj0h`+pX8cJt;&B!U*mew>*;@1uqZe0XDudJ-R
zyX@_*>hJH4c8i0SRMr3csou_1V8ZY%cxzj?nC_zphp)>|H*^WAg?B%Xt&BbOfP3PD
zfY{jB(9qE0Kc849M!B*T1XM8v&;6otAVbu!W?{h7s|&n8#5Hv4cRb0IEUDUSXJVpb
zXIFo5(TTjfD|SEo!6BhBC48|f%LiAxrAa*B!q>%AzPYin?Cq_$H#RPI@0a_wwf4h7
zcFUTlE0@pP)L$YQc2CkIaw+ppAK?!ZugK2koRVcUvuTdYi}1?I%0HrZt)4GM(>6yM
zu3`NX?Qq^)O}(8*wyNasuZZ4di+0RdzdGpksZ(BYy$*{Pllp`LmB^p?~G;FO>3z?>Oco$T_p!BTy+=*eTtZftWj#A%a=b6~kG-!$*15UX={{GMcxRn>sTsPczy9GN
z)-;!ed_RINFZZ`kJ0r1pxrXNAKk{`y6#H8;V?VAtrmEyN>ug;A?lT#F3JlS8g%3YD
z*>E|TwjOf$pwPae=4VjKIhoDX<=bQ|*Qk96u(-OSWlpl({jj}NUvF*C&zH~7>GGLr
zw9)f>{o7kxH>aO>^D(@5ykGu(>iQK=g%b{}v{pNCxi!aVme|4vuNV$C%LFa)o4YXQ
z=H&l1fu|g8c`bjNaJm
z={b)@1sDYd8kY#12-^K^V^-v7V$-2q0slnIa2
zj>lzBnrBtrJ7qRUheL(HiHHpkmO8he`~Tzd*6D3BzuwLIVq&vyua_p1(gU5wjyE?i
zi)oi7ea%`EynJ7>rfumrDNT@V8x$uf`p>&>o|AA_`eo*g7-P{878eB*&Xme87mE!I
z>yI5-($RBd;;b)Dr|f*KoS!N%DG5wtS;Qr>Hs!Mn?r_EgX8cM0{WnxzrrrNgB3
zK#ocC^0MAlvA@4OpFcabU{ajn%gjrXTI_-XjRmR`mIf@mJioT=Z0jo*-X+WZ0z%}J
z6qp`*K6rL&>VJ#sZ|4`g%WCfXaz(J`lnaZCf)1z4vy+qc)SkU*Kgu#l3M8z+bkaxRy0HJ8AJ3%S{j_XLSr!`!G755pGp#Io
zer}4f`sw*9wPE{z{eD;_yyA|M0@FEx9R&|Ry}aC)q6$+i&{*&3Un<38(Ad2anz=U~LhXZ$lReTBg4pPWr;p|ZLipTw_5|few)1~0M%pfff
zco<)tZaczrC?v#B{r0^Spmwug7Z!@7mea82c!D
zW!|or%Q`wb4qV=Cs`{N-=+X=rt?3c8N^8_w@Ai>Thp$R)5cv7EU;D
zw#ZpoSb_Dry7z3ewX?*Er(Ry_{ruNy*#y?hOFV@Wyvo(Rr)q`%`tq{&wS=T(-8B2y(NdlbSteSb3hmp8
ztEj9QsvRrTffmQQ3
zIvh3$PMD`g74d@2^X6HVzWVU+@X59ybuMnEJ3{q;KDtMUgtq#aF|Gdp@8)~&yB{ip
zzrMTsyN#dkBlFKqS8i-fuBasg0YV@Dq*;!oE+Vg+H?)8&bJ3Ku#b#>UF7;aj*HLlBj!s
z%0I3ze0;1%s_gPIU%hq7HlfL_O*3Z92wfd^c7|c{!6w!lsap&~SBE|QZ23{<>ATqd
zf4|+<-G1j#xvu{_o0soq9SgNS!>#i8sCfJq&TTn2FD>(x-u-grftQz;zuc6N74TZw
zy)R@<#K#8*o6UU=oV`DJUEJPZ>GNybY~$BNY*g85nOI?9WR-hs%irJMl^3XbPkUng
zc+HCod#lT%rr+G0UcGI8_l##}XPd|GtGT^Bpa1acR}K?zZ_VDGbhK-lZuF&-Z!a(R
z@0YVJkh-=$et*f!OMFk{+9L1#`F>tRW^?lEYim7UoxZn0sQpKyY1Wm7i{|%hgdaWS
z3Ho?$j^*X0-qVkEi{Jj^Jyol9Hc$Sh-j7iarpMPkyt#1y`>kEKe=S;KBXVLz=gn=o
zw_mT{e@^#c?V(k2W;5BDfB3uiO0mvSpI31xr2g8PNbT@-J~~|AYu593-?f^Qng9Ik
z?C)>4-xt%5yR*ec?$2_2>#8pqGR~}AA`U(;uV2`ea%#%SV2uwV(#e0W$Jc|_Gt4c&
zcT&h=pZ8v)b=~SMU%6hM{ZMQu^X7t9=qeH2sFF)dJeSS#DRr=$rW5I;e(b2|_tm%h
zWUUKk2yssEp02lWM#(d~|9^^=vahYNtp8V&(qbp`&Dw9_k#EU&7BA=7oUpq5#RW!I
z&>GRq{GFwL0;9w`d?yRq`tV~YMX;{82V4+jV
zvyRAJC7N8`(Wmbw2_KO!=&$8csN>mLaR0)2)7f$Vj(fbnv(s3%W>50z>H5dbK0cFi
zRx>uqy0XHxTkK2j*}n(6<$OHV&ir6L9d+UM+UV_iX&%e{<`&EhT^Dn6?RMTY@vBGt
z+^r^hTm7?W+7L0b>SV}5oybiOgbwE4SRZfyrs^`^PEpn{%i^OMsqvLhMMau#H@|!}
z}^*2x1c;oQKCwDbw*+>vRx?7-HntB-CPBu}llFoA!K@v}yQ`DeaPsr&Je{n&zE-`}5q
z!DO<*hK*k?=ZvA*M?OckxxCZ0XHO|gN>P|n@~W`=>|E>V8)WyVF6rCNblpAa^Z)Q9
zevZFiE?8E5$#^l{xYDbGvn@j9{7$EzIuf#OrUqvv`VYO>u=J=-?dP-RZ`d2?G>30dCPBOWaC(M=mVXS7sJ8V>xUj%+
zhocTl%$YBrBsOg}l;gQD|M6ReBPFk{tdv`0C;YPN>#L{Hg>z2tu}^dR|8moXV-Hvl
znIDSg|G0bVU7Pvye>>X?eo#KJW3B8#cb}Po$G6;B<~#eB>gN+`vA^BrDxcignA~&t
zfkE!QJv-gC-bMRX{VIw7(DLCH_eTp`HXexv^_DswG408WGjiPd+g_IVoJvtr+p$Lf
zp1J&?Jn8;VtLK@y9_VLqV;8<9+5hNfLP6U{jkX^%7rmZ-MtOEmz{f6;#eagA`5cUm
z*in!ex8sY)-P+Zi<}<%@O3p|Ulbvq_+LZlQLTz&5QX?Ppx<3_Bb5ES`;Cskg|77Y)
zmwoNGH+;FWDByrL%Z%Rc8iQvC7MR7KNxU{CJgaVR)uqOxKctM)c!UjRwn&SLmhKIC
z*MIock?rSPoOAr|{a&y$^^wRrhG+H(2R3z|vzvG(_1(3#(#F%*d%ILs#p-m4F!BpN
zh?Pj=FjRZI&`LcdRITO0F$0yOUyLppHFQpweEj9YLT8JP4?E+{nSc7P?{=N2^gxk`
zXWNI*0j8EJS&(|=1+bCY}(|D_;@Wk
zKULQ(=+oA+Y5YunHtw0z%{foleSKE?KYneb;Pmx<;j6>ehO7))(Uy8Js{jAb^YyQ;
zt-ZaY@G@teezhZbmaV^=jY~b&AR&QQMdjhtD5`!
zYIRC)ZqJVw6TY{%`um%knnRzDUWP@AL#$x
zn?+u4t^WS5W1sHPJ?ki;VUZJw3H4^)#QH&5RX`r&xHf
zq|C3{WxDiJNvxrtxPIK3kTa`O7#2MOmm(^?~|9vooY
zGHJ@?WxmNvzusuSFfC+d(9)`}ubA2SzUZoRZ`qo0QAtTj>6>C_Z{+s8yAKby3s;A{
z`SOD4(wSd{8_PA-esX&FPo7_yG56!8rQR08FK=C69lqXsx?Y6UwwmYX=ko_mzavpw
zsGF3>Ezra8P17v)Pu=Z*e}B)nsho7L_6?JRX8gVy&nx;a1!rfOS{6U+nOmQfyd!Rl
zz_Ln@Hygd)TQvTkf05z%>V-V{GZ@aBHT%iOYxqS5Tyy0qDI74aKm#v02k>0l3Ry>rJwXHH)n!4l6vovNN=g>6~f_!Ri
z%Q$w=`?MjT`tPr=%F4=}?A+N~Yrl)b$*(78%c`mfo`;ag_F$2Rw8n!(GSNIr{{
zwXd@|Z~TAb>D~eZgR84TSLfc|wo!cJ4C8da*=D&tKNgASJj!+HoWS2UdxqtO6@iOI
zwZqQLx1Ybym`ziv(Qek9K$DczBOQV#jxh7vH0%Iv9$L|CoB#aW+>4i=ot>>L;K8QU
z#G-sT;(~n5hl9_~&JJH6cXvagv&vz)N74@QQYIM=pD%1m^=@IaUn%M6R3%VeA|D#N
zA%Su02isFEg6>{ZmEHSp?C$WKtmgav-rlo8(?ixexAV1fi8667uKc51vLw-np*u-w
zmUBDbT&vQoe}8^vdc3-_^6<)=-B&Ihb4Yl4YU+~%3DYevwr~ntEJ-^%Yl|n#3bl_*
zQe=3JdoTB!Yhh|=Tm6kiC%MWbvqSBP<=szDPoF&1qv0esV~eowER%%?7lg#uew_*m
zJimtBzV`aU4jB@%No7+S4;(&Ydp!7*+PjdPAnxNDnnjsO27f&)Yk!q&?o>+VeHdpl
zJ7tDtahl1MoFpb??Vg_5XA2)6OWd{SZrbGkQl?oql0mh;$gS9BHr_*@jAozUP4bnn
zJ-X9#o7qg>fJuv7yB|q!HT-ht>Os#I^BxiB-ypyK7^Sq5>Zzr4JBncsA_&3}%T
zx6g$-B@}sISUlW*zfStA;sMPKs)E5sxQlxv&h@<$a0oSQ_x(T1_rCm3g9T4sJ1m%Z
zz~oqy^39pX=_ick_8SN%d#0^^IeT-y#BF0Oty5o4`P!sCKGe#+J^%hWH8H(yT8lE5
zuQb}1zBY5$Pk|4*%(-lDdF_=FTW3sSEZ(Qp$SLal*HhDQzNrmirINh?}w&)o1&x`Kz4{v>5%fMT0Gh2hBHez<0
zugb|)LPEzjsu)-*++V@wlwj;Kn?u>ngW+xgORJx)Y|WX(zD3=+DN{_2acA=P$Ubl=
z5&OAqcR&iyJ=^MUHzcC{q?4I{L}z-u_pPFv8>OBS
znPavk_sPi{Q8D|t@7IWwwH)~WF5mrAklH&1hxbL#&dk)`^MQ%kajEz8ZBswh{_ETl
zl_@On!Ta&{m$N5rFztHV*rB$l`Afp>q+f2
zF6hPYvk5w`8ZswI4H_-ow%FrG$Z>O!7XktjH+&J%Z=Xe
zyhW0SX_J4mqv@PaPZ-^fo{4DJH9zMrv*63$-14jMHp$lL1SqX-OuKR_>sqP#+G`Uy
z6RUN8N>mH{XE=4}hv+2-mDIy$R3tAmeE7WSWp#({r1*xw(qF&R7*4%8^ycX0Q#)*C
zFXP!Zr|$Tyvw3%SF)s0+Z@2NgQu-8uq!rT4U-s3Oadlpst;zmOXWKqC`Q4t)j$W&r
zZ6>uZS@gi7j91L)xyTNS4ny{)SvSRFME30J$&TF4S>4;4uM^J`VIuoqUsyV0!Ut2(
z@k@`6bZ+5QlmE*;yWxw;jTUFy`j)om*YcN0cc-10pm@V&UsdCAVav^POWFcf%$^#q
z((q+Q&ay)ttjBNpek)R{)=`s4O8MHP>1gwFvbw+7CcBCc30IEva8Hyzbz(tRVfL(t
zZ+^^7nq{b9oN$0)HV;4B=UuD9)_V15ZkTE|`V2K27v*2HV_@
z6gH;fg&xVm7f!#su~E4>K>h3o_&4Y4vP;bZ{4vl
zopOUkD*4NeC0lYmZ2Gv>%x-f=3Yd~taBf7Y%KHjkgCoM6r7Slw0O(kQ3lXI1s|
zg?O0a;vkj77LWEh=?4hN6-{aKt!a9;@p#kWK#%fyzR&m#&h%=T=H8leJ^E~CKygjI
z%cpBL!n0>&ojT5P;j)}<)rISB(=F%tsxcpBo%zvZjq4c`H4jkV%Sho`hD>ZiT8ZMR
z85JECPRTmEZ~Qbc>Cag>PeDnw^kD9r{sg7`b4KMg3gl`e^3-~);@GWGSzMC`+dLjLZ;fx
zOmeM0FhL~B)6ihMCv(=h6FzLWoMP5Voa6cOIHpbg`O>wIV)lG<^J3e*Luupd6%xle
z{~XqQSMlzY!4XacZJQ?=3WwS^MZDZ{^w_SAVkLeX8T-dHYr!tU3EK;!3
zMJc-E(nX!N%_n5!l^BAJ;uoL^!7>-5VTN8fnAu}DhlpQ*rpW5bg+&NfzePYeBB
z2LgL)C*LbSa#HMudBd4NEtb;tD1zI-QKW6kJ!u@U2R5*#(2EDvZw`Ii$G=JaA>^+s+!M
zu!3dDa>t{q&$v3U*tj_S3+9i_2@Pd&K@+cfwXl2bs;&kOmZ=Bo8CM^3dLeTtZyf^z
O1B0ilpUXO@geCw`{*0Rd

literal 0
HcmV?d00001


From 33d643124d8908d0efea03f4c77d450423ac75e9 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Thu, 4 Oct 2018 17:25:11 -0700
Subject: [PATCH 372/372] fixed fullbench-lib target

---
 build/VS2008/fullbench/fullbench.vcproj | 4 ++++
 build/cmake/tests/CMakeLists.txt        | 8 ++++----
 tests/Makefile                          | 6 ++++--
 3 files changed, 12 insertions(+), 6 deletions(-)

diff --git a/build/VS2008/fullbench/fullbench.vcproj b/build/VS2008/fullbench/fullbench.vcproj
index 2ce7f74e7..a31883acf 100644
--- a/build/VS2008/fullbench/fullbench.vcproj
+++ b/build/VS2008/fullbench/fullbench.vcproj
@@ -388,6 +388,10 @@
 				RelativePath="..\..\..\programs\datagen.c"
 				>
 			
+			
+			
 			
diff --git a/build/cmake/tests/CMakeLists.txt b/build/cmake/tests/CMakeLists.txt
index 11c0db140..9f4c64cf6 100644
--- a/build/cmake/tests/CMakeLists.txt
+++ b/build/cmake/tests/CMakeLists.txt
@@ -40,7 +40,10 @@ SET(PROGRAMS_DIR ${ZSTD_SOURCE_DIR}/programs)
 SET(TESTS_DIR ${ZSTD_SOURCE_DIR}/tests)
 INCLUDE_DIRECTORIES(${TESTS_DIR} ${PROGRAMS_DIR} ${LIBRARY_DIR} ${LIBRARY_DIR}/common ${LIBRARY_DIR}/compress ${LIBRARY_DIR}/dictBuilder)
 
-ADD_EXECUTABLE(fullbench ${PROGRAMS_DIR}/datagen.c ${TESTS_DIR}/fullbench.c)
+ADD_EXECUTABLE(datagen ${PROGRAMS_DIR}/datagen.c ${TESTS_DIR}/datagencli.c)
+TARGET_LINK_LIBRARIES(datagen libzstd_static)
+
+ADD_EXECUTABLE(fullbench ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/bench.c ${TESTS_DIR}/fullbench.c)
 TARGET_LINK_LIBRARIES(fullbench libzstd_static)
 
 ADD_EXECUTABLE(fuzzer ${PROGRAMS_DIR}/datagen.c ${TESTS_DIR}/fuzzer.c)
@@ -49,7 +52,4 @@ TARGET_LINK_LIBRARIES(fuzzer libzstd_static)
 IF (UNIX)
     ADD_EXECUTABLE(paramgrill ${PROGRAMS_DIR}/bench.c ${PROGRAMS_DIR}/datagen.c ${TESTS_DIR}/paramgrill.c)
     TARGET_LINK_LIBRARIES(paramgrill libzstd_static m) #m is math library
-
-    ADD_EXECUTABLE(datagen ${PROGRAMS_DIR}/datagen.c ${TESTS_DIR}/datagencli.c)
-    TARGET_LINK_LIBRARIES(datagen libzstd_static)
 ENDIF (UNIX)
diff --git a/tests/Makefile b/tests/Makefile
index 50c1a6156..2a96829f6 100644
--- a/tests/Makefile
+++ b/tests/Makefile
@@ -81,7 +81,8 @@ DECODECORPUS_TESTTIME ?= -T30
 default: fullbench
 	@echo $(ZSTDMT_OBJECTS)
 
-all: fullbench fuzzer zstreamtest paramgrill datagen decodecorpus roundTripCrash
+all: fullbench fuzzer zstreamtest paramgrill datagen decodecorpus roundTripCrash \
+     fullbench-lib
 
 all32: fullbench32 fuzzer32 zstreamtest32
 
@@ -134,8 +135,9 @@ fullbench fullbench32 : $(ZSTD_FILES)
 fullbench fullbench32 : $(PRGDIR)/datagen.c $(PRGDIR)/bench.c fullbench.c
 	$(CC) $(FLAGS) $^ -o $@$(EXT)
 
+fullbench-lib : CPPFLAGS += -DXXH_NAMESPACE=ZSTD_
 fullbench-lib : zstd-staticLib
-fullbench-lib : $(PRGDIR)/datagen.c fullbench.c
+fullbench-lib : $(PRGDIR)/datagen.c $(PRGDIR)/bench.c fullbench.c
 	$(CC) $(FLAGS) $(filter %.c,$^) -o $@$(EXT) $(ZSTDDIR)/libzstd.a
 
 # note : broken : requires unavailable symbols