Merge pull request #1236 from GeorgeLu97/paramgrillconstraints

ParamgrillConstraints

programs/bench.c

@@ -171,6 +171,8 @@ struct  BMK_timeState_t{
 static void BMK_initCCtx(ZSTD_CCtx* ctx, 
     const void* dictBuffer, size_t dictBufferSize, int cLevel, 
     const ZSTD_compressionParameters* comprParams, const BMK_advancedParams_t* adv) {
+    ZSTD_CCtx_reset(ctx);
+    ZSTD_CCtx_resetParameters(ctx);
     if (adv->nbWorkers==1) {
         ZSTD_CCtx_setParameter(ctx, ZSTD_p_nbWorkers, 0);
     } else {
@@ -195,6 +197,7 @@ static void BMK_initCCtx(ZSTD_CCtx* ctx,
 
 static void BMK_initDCtx(ZSTD_DCtx* dctx, 
     const void* dictBuffer, size_t dictBufferSize) {
+    ZSTD_DCtx_reset(dctx);
     ZSTD_DCtx_loadDictionary(dctx, dictBuffer, dictBufferSize);
 }
 
@@ -291,34 +294,34 @@ BMK_customReturn_t BMK_benchFunction(
     BMK_initFn_t initFn, void* initPayload,
     size_t blockCount,
     const void* const * const srcBlockBuffers, const size_t* srcBlockSizes,
-    void* const * const dstBlockBuffers, const size_t* dstBlockCapacities,
+    void* const * const dstBlockBuffers, const size_t* dstBlockCapacities, size_t* blockResult, 
     unsigned nbLoops) {
-    size_t srcSize = 0, dstSize = 0, ind = 0;
+    size_t dstSize = 0;
     U64 totalTime;
 
     BMK_customReturn_t retval;
     UTIL_time_t clockStart;
 
-    {
-        unsigned i;
-        for(i = 0; i < blockCount; i++) {
-            memset(dstBlockBuffers[i], 0xE5, dstBlockCapacities[i]);  /* warm up and erase result buffer */
-        }
-
-        UTIL_sleepMilli(5);  /* give processor time to other processes */
-        UTIL_waitForNextTick();
-    }
-
     if(!nbLoops) {
         EXM_THROW_ND(1, BMK_customReturn_t, "nbLoops must be nonzero \n");
     }
 
-    for(ind = 0; ind < blockCount; ind++) {
+    {
-        srcSize += srcBlockSizes[ind];
+        size_t i;
+        for(i = 0; i < blockCount; i++) {
+            memset(dstBlockBuffers[i], 0xE5, dstBlockCapacities[i]);  /* warm up and erase result buffer */
+        }
+#if 0
+        /* based on testing these seem to lower accuracy of multiple calls of 1 nbLoops vs 1 call of multiple nbLoops 
+         * (Makes former slower)   
+         */
+        UTIL_sleepMilli(5);  /* give processor time to other processes */
+        UTIL_waitForNextTick();
+#endif
     }
 
     {      
-        unsigned i, j, firstIter = 1;
+        unsigned i, j;
         clockStart = UTIL_getTime();
         if(initFn != NULL) { initFn(initPayload); }
         for(i = 0; i < nbLoops; i++) {
@@ -327,11 +330,13 @@ BMK_customReturn_t BMK_benchFunction(
                 if(ZSTD_isError(res)) {
                     EXM_THROW_ND(2, BMK_customReturn_t, "Function benchmarking failed on block %u of size %u : %s  \n",
                         j, (U32)dstBlockCapacities[j], ZSTD_getErrorName(res));
-                } else if(firstIter) {
+                } else if(i == nbLoops - 1) {
                     dstSize += res;
-                }
+                    if(blockResult != NULL) {
+                        blockResult[j] = res;
+                    }
+                } 
             }
-            firstIter = 0;
         }
         totalTime = UTIL_clockSpanNano(clockStart);
     }
@@ -353,6 +358,9 @@ void BMK_resetTimeState(BMK_timedFnState_t* r, unsigned nbSeconds) {
 
 BMK_timedFnState_t* BMK_createTimeState(unsigned nbSeconds) {
     BMK_timedFnState_t* r = (BMK_timedFnState_t*)malloc(sizeof(struct BMK_timeState_t));
+    if(r == NULL) {
+        return r;
+    }
     BMK_resetTimeState(r, nbSeconds);
     return r;
 }
@@ -361,13 +369,14 @@ void BMK_freeTimeState(BMK_timedFnState_t* state) {
     free(state);
 }
 
+/* make option for dstBlocks to be */
 BMK_customTimedReturn_t BMK_benchFunctionTimed(
     BMK_timedFnState_t* cont,
     BMK_benchFn_t benchFn, void* benchPayload,
     BMK_initFn_t initFn, void* initPayload,
     size_t blockCount,
     const void* const* const srcBlockBuffers, const size_t* srcBlockSizes,
-    void* const* const dstBlockBuffers, const size_t* dstBlockCapacities) 
+    void * const * const dstBlockBuffers, const size_t * dstBlockCapacities, size_t* blockResults) 
 {
     U64 fastest = cont->fastestTime;
     int completed = 0;
@@ -382,9 +391,9 @@ BMK_customTimedReturn_t BMK_benchFunctionTimed(
             UTIL_sleep(COOLPERIOD_SEC);
             cont->coolTime = UTIL_getTime();
         }
-
+        /* reinitialize capacity */
         r.result = BMK_benchFunction(benchFn, benchPayload, initFn, initPayload,
-        blockCount, srcBlockBuffers, srcBlockSizes, dstBlockBuffers, dstBlockCapacities, cont->nbLoops);
+        blockCount, srcBlockBuffers, srcBlockSizes, dstBlockBuffers, dstBlockCapacities, blockResults, cont->nbLoops);
         if(r.result.error) { /* completed w/ error */
             r.completed = 1;
             return r;
@@ -393,7 +402,7 @@ BMK_customTimedReturn_t BMK_benchFunctionTimed(
         {   U64 const loopDuration = r.result.result.nanoSecPerRun * cont->nbLoops;
             r.completed = (cont->timeRemaining <= loopDuration);
             cont->timeRemaining -= loopDuration;
-            if (loopDuration > 0) { 
+            if (loopDuration > (TIMELOOP_NANOSEC / 100)) { 
                 fastest = MIN(fastest, r.result.result.nanoSecPerRun);
                 if(loopDuration >= MINUSABLETIME) {
                     r.result.result.nanoSecPerRun = fastest;
@@ -418,9 +427,9 @@ BMK_customTimedReturn_t BMK_benchFunctionTimed(
 /* benchMem with no allocation */
 static BMK_return_t BMK_benchMemAdvancedNoAlloc(
     const void ** const srcPtrs, size_t* const srcSizes,
-    void** const cPtrs, size_t* const cSizes,
+    void** const cPtrs, size_t* const cCapacities, size_t* const cSizes, 
     void** const resPtrs, size_t* const resSizes,
-    void* resultBuffer, void* compressedBuffer,
+    void** resultBufferPtr, void* compressedBuffer,
     const size_t maxCompressedSize,
     BMK_timedFnState_t* timeStateCompress, BMK_timedFnState_t* timeStateDecompress,
 
@@ -432,7 +441,7 @@ static BMK_return_t BMK_benchMemAdvancedNoAlloc(
     int displayLevel, const char* displayName, const BMK_advancedParams_t* adv) 
 {
     size_t const blockSize = ((adv->blockSize>=32 && (adv->mode != BMK_decodeOnly)) ? adv->blockSize : srcSize) + (!srcSize); /* avoid div by 0 */
-    BMK_return_t results;
+    BMK_return_t results = { { 0, 0, 0, 0 }, 0 } ;
     size_t const loadedCompressedSize = srcSize;
     size_t cSize = 0;
     double ratio = 0.;
@@ -454,13 +463,13 @@ static BMK_return_t BMK_benchMemAdvancedNoAlloc(
             srcPtr += fileSizes[fileNb];
         }
         {   size_t const decodedSize = (size_t)totalDSize64;
-            free(resultBuffer);
+            free(*resultBufferPtr);
-            resultBuffer = malloc(decodedSize);
+            *resultBufferPtr = malloc(decodedSize);
-            if (!resultBuffer) { 
+            if (!(*resultBufferPtr)) {
                 EXM_THROW(33, BMK_return_t, "not enough memory"); 
             }
             if (totalDSize64 > decodedSize) {
-                free(resultBuffer); 
+                free(*resultBufferPtr); 
                 EXM_THROW(32, BMK_return_t, "original size is too large");   /* size_t overflow */
             }
             cSize = srcSize;
@@ -472,7 +481,7 @@ static BMK_return_t BMK_benchMemAdvancedNoAlloc(
     /* Init data blocks  */
     {   const char* srcPtr = (const char*)srcBuffer;
         char* cPtr = (char*)compressedBuffer;
-        char* resPtr = (char*)resultBuffer;
+        char* resPtr = (char*)(*resultBufferPtr);
         U32 fileNb;
         for (nbBlocks=0, fileNb=0; fileNb<nbFiles; fileNb++) {
             size_t remaining = fileSizes[fileNb];
@@ -483,11 +492,11 @@ static BMK_return_t BMK_benchMemAdvancedNoAlloc(
                 srcPtrs[nbBlocks] = (const void*)srcPtr;
                 srcSizes[nbBlocks] = thisBlockSize;
                 cPtrs[nbBlocks] = (void*)cPtr;
-                cSizes[nbBlocks] = (adv->mode == BMK_decodeOnly) ? thisBlockSize : ZSTD_compressBound(thisBlockSize);
+                cCapacities[nbBlocks] = (adv->mode == BMK_decodeOnly) ? thisBlockSize : ZSTD_compressBound(thisBlockSize);
                 resPtrs[nbBlocks] = (void*)resPtr;
                 resSizes[nbBlocks] = (adv->mode == BMK_decodeOnly) ? (size_t) ZSTD_findDecompressedSize(srcPtr, thisBlockSize) : thisBlockSize;
                 srcPtr += thisBlockSize;
-                cPtr += cSizes[nbBlocks];
+                cPtr += cCapacities[nbBlocks];
                 resPtr += thisBlockSize;
                 remaining -= thisBlockSize;
             }   
@@ -538,7 +547,7 @@ static BMK_return_t BMK_benchMemAdvancedNoAlloc(
                 while(!(intermediateResultCompress.completed && intermediateResultDecompress.completed)) {
                     if(!intermediateResultCompress.completed) { 
                         intermediateResultCompress = BMK_benchFunctionTimed(timeStateCompress, &local_defaultCompress, (void*)ctx, &local_initCCtx, (void*)&cctxprep,
-                        nbBlocks, srcPtrs, srcSizes, cPtrs, cSizes);
+                        nbBlocks, srcPtrs, srcSizes, cPtrs, cCapacities, cSizes);
                         if(intermediateResultCompress.result.error) {
                             results.error = intermediateResultCompress.result.error;
                             return results;
@@ -546,9 +555,7 @@ static BMK_return_t BMK_benchMemAdvancedNoAlloc(
                         ratio = (double)(srcSize / intermediateResultCompress.result.result.sumOfReturn);
                         {   
                             int const ratioAccuracy = (ratio < 10.) ? 3 : 2;
-                            double const compressionSpeed = ((double)srcSize / intermediateResultCompress.result.result.nanoSecPerRun) * 1000;
+                            results.result.cSpeed = (srcSize * TIMELOOP_NANOSEC / intermediateResultCompress.result.result.nanoSecPerRun);
-                            int const cSpeedAccuracy = (compressionSpeed < 10.) ? 2 : 1;
-                            results.result.cSpeed = compressionSpeed * 1000000;
                             cSize = intermediateResultCompress.result.result.sumOfReturn;
                             results.result.cSize = cSize;
                             ratio = (double)srcSize / results.result.cSize;
@@ -556,13 +563,13 @@ static BMK_return_t BMK_benchMemAdvancedNoAlloc(
                             DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.*f),%6.*f MB/s\r",
                                     marks[markNb], displayName, (U32)srcSize, (U32)results.result.cSize,
                                     ratioAccuracy, ratio,
-                                    cSpeedAccuracy, compressionSpeed);
+                                    results.result.cSpeed < (10 MB) ? 2 : 1, (double)results.result.cSpeed / (1 MB));
                         }
                     }
 
                     if(!intermediateResultDecompress.completed) {
                         intermediateResultDecompress = BMK_benchFunctionTimed(timeStateDecompress, &local_defaultDecompress, (void*)(dctx), &local_initDCtx, (void*)&dctxprep,
-                        nbBlocks, (const void* const*)cPtrs, cSizes, resPtrs, resSizes);
+                        nbBlocks, (const void* const*)cPtrs, cSizes, resPtrs, resSizes, NULL);
                         if(intermediateResultDecompress.result.error) {
                             results.error = intermediateResultDecompress.result.error;
                             return results;
@@ -570,24 +577,22 @@ static BMK_return_t BMK_benchMemAdvancedNoAlloc(
       
                         {   
                             int const ratioAccuracy = (ratio < 10.) ? 3 : 2;
-                            double const compressionSpeed = results.result.cSpeed / 1000000;
+                            results.result.dSpeed = (srcSize * TIMELOOP_NANOSEC/ intermediateResultDecompress.result.result.nanoSecPerRun);
-                            int const cSpeedAccuracy = (compressionSpeed < 10.) ? 2 : 1;
-                            double const decompressionSpeed = ((double)srcSize / intermediateResultDecompress.result.result.nanoSecPerRun) * 1000;
-                            results.result.dSpeed = decompressionSpeed * 1000000;
                             markNb = (markNb+1) % NB_MARKS;
                             DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.*f),%6.*f MB/s ,%6.1f MB/s \r",
                                     marks[markNb], displayName, (U32)srcSize, (U32)results.result.cSize,
                                     ratioAccuracy, ratio,
-                                    cSpeedAccuracy, compressionSpeed,
+                                    results.result.cSpeed < (10 MB) ? 2 : 1, (double)results.result.cSpeed / (1 MB),
-                                    decompressionSpeed);
+                                    (double)results.result.dSpeed / (1 MB));
                         }
                     }
                 }
 
-            } else {
+            } else { //iterMode;
                 if(adv->mode != BMK_decodeOnly) {
+
                     BMK_customReturn_t compressionResults = BMK_benchFunction(&local_defaultCompress, (void*)ctx, &local_initCCtx, (void*)&cctxprep,
-                        nbBlocks, srcPtrs, srcSizes, cPtrs, cSizes, adv->nbSeconds);
+                        nbBlocks, srcPtrs, srcSizes, cPtrs, cCapacities, cSizes, adv->nbSeconds); 
                     if(compressionResults.error) {
                         results.error = compressionResults.error;
                         return results;
@@ -595,26 +600,27 @@ static BMK_return_t BMK_benchMemAdvancedNoAlloc(
                     if(compressionResults.result.nanoSecPerRun == 0) {
                         results.result.cSpeed = 0;
                     } else {
-                        results.result.cSpeed = (double)srcSize / compressionResults.result.nanoSecPerRun * TIMELOOP_NANOSEC;
+                        results.result.cSpeed = srcSize * TIMELOOP_NANOSEC / compressionResults.result.nanoSecPerRun;
                     }
                     results.result.cSize = compressionResults.result.sumOfReturn;
                     {   
                         int const ratioAccuracy = (ratio < 10.) ? 3 : 2;
-                        double const compressionSpeed = results.result.cSpeed / 1000000;
+                        results.result.cSpeed = (srcSize * TIMELOOP_NANOSEC / compressionResults.result.nanoSecPerRun);
-                        int const cSpeedAccuracy = (compressionSpeed < 10.) ? 2 : 1;
+                        cSize = compressionResults.result.sumOfReturn;
+                        results.result.cSize = cSize;
                         ratio = (double)srcSize / results.result.cSize;
                         markNb = (markNb+1) % NB_MARKS;
                         DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.*f),%6.*f MB/s\r",
                                 marks[markNb], displayName, (U32)srcSize, (U32)results.result.cSize,
                                 ratioAccuracy, ratio,
-                                cSpeedAccuracy, compressionSpeed);
+                                results.result.cSpeed < (10 MB) ? 2 : 1, (double)results.result.cSpeed / (1 MB));
                     }
                 }
                 if(adv->mode != BMK_compressOnly) {
                     BMK_customReturn_t decompressionResults = BMK_benchFunction(
                         &local_defaultDecompress, (void*)(dctx),
                         &local_initDCtx, (void*)&dctxprep, nbBlocks,
-                        (const void* const*)cPtrs, cSizes, resPtrs, resSizes, 
+                        (const void* const*)cPtrs, cSizes, resPtrs, resSizes, NULL, 
                         adv->nbSeconds);
                     if(decompressionResults.error) {
                         results.error = decompressionResults.error;
@@ -623,26 +629,26 @@ static BMK_return_t BMK_benchMemAdvancedNoAlloc(
                     if(decompressionResults.result.nanoSecPerRun == 0) {
                         results.result.dSpeed = 0;
                     } else {
-                        results.result.dSpeed = (double)srcSize / decompressionResults.result.nanoSecPerRun * TIMELOOP_NANOSEC;
+                        results.result.dSpeed = srcSize * TIMELOOP_NANOSEC / decompressionResults.result.nanoSecPerRun;
                     }
-                    {   int const ratioAccuracy = (ratio < 10.) ? 3 : 2;
+
-                        double const compressionSpeed = results.result.cSpeed / 1000000;
+                    {   
-                        int const cSpeedAccuracy = (compressionSpeed < 10.) ? 2 : 1;
+                        int const ratioAccuracy = (ratio < 10.) ? 3 : 2;
-                        double const decompressionSpeed = ((double)srcSize / decompressionResults.result.nanoSecPerRun) * 1000;
+                        results.result.dSpeed = (srcSize * TIMELOOP_NANOSEC/ decompressionResults.result.nanoSecPerRun);
-                        results.result.dSpeed = decompressionSpeed * 1000000;
                         markNb = (markNb+1) % NB_MARKS;
                         DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.*f),%6.*f MB/s ,%6.1f MB/s \r",
                                 marks[markNb], displayName, (U32)srcSize, (U32)results.result.cSize,
                                 ratioAccuracy, ratio,
-                                cSpeedAccuracy, compressionSpeed,
+                                results.result.cSpeed < (10 MB) ? 2 : 1, (double)results.result.cSpeed / (1 MB),
-                                decompressionSpeed);
+                                (double)results.result.dSpeed / (1 MB));
                     }
                 }
             }
         }
 
         /* CRC Checking */
-        {   U64 const crcCheck = XXH64(resultBuffer, srcSize, 0);
+        {   void* resultBuffer = *resultBufferPtr;
+            U64 const crcCheck = XXH64(resultBuffer, srcSize, 0);
             /* adv->mode == 0 -> compress + decompress */
             if ((adv->mode == BMK_both) && (crcOrig!=crcCheck)) {
                 size_t u;
@@ -682,8 +688,8 @@ static BMK_return_t BMK_benchMemAdvancedNoAlloc(
         }   /* CRC Checking */
 
     if (displayLevel == 1) {   /* hidden display mode -q, used by python speed benchmark */
-        double const cSpeed = results.result.cSpeed / 1000000;
+        double const cSpeed = (double)results.result.cSpeed / (1 MB);
-        double const dSpeed = results.result.dSpeed / 1000000;
+        double const dSpeed = (double)results.result.dSpeed / (1 MB);
         if (adv->additionalParam) {
             DISPLAY("-%-3i%11i (%5.3f) %6.2f MB/s %6.1f MB/s  %s (param=%d)\n", cLevel, (int)cSize, ratio, cSpeed, dSpeed, displayName, adv->additionalParam);
         } else {
@@ -692,14 +698,16 @@ static BMK_return_t BMK_benchMemAdvancedNoAlloc(
     }
     DISPLAYLEVEL(2, "%2i#\n", cLevel);
     }   /* Bench */
+    results.result.cMem = (1ULL << (comprParams->windowLog)) + ZSTD_sizeof_CCtx(ctx);
+    results.error = 0;
     return results;
 }
 
 BMK_return_t BMK_benchMemAdvanced(const void* srcBuffer, size_t srcSize,
+                        void* dstBuffer, size_t dstCapacity, 
                         const size_t* fileSizes, unsigned nbFiles,
                         const int cLevel, const ZSTD_compressionParameters* comprParams,
                         const void* dictBuffer, size_t dictBufferSize,
-                        ZSTD_CCtx* ctx, ZSTD_DCtx* dctx,
                         int displayLevel, const char* displayName, const BMK_advancedParams_t* adv)
 
 {
@@ -707,47 +715,70 @@ BMK_return_t BMK_benchMemAdvanced(const void* srcBuffer, size_t srcSize,
     U32 const maxNbBlocks = (U32) ((srcSize + (blockSize-1)) / blockSize) + nbFiles;
 
     /* these are the blockTable parameters, just split up */
-    const void ** const srcPtrs = (const void** const)malloc(maxNbBlocks * sizeof(void*));
+    const void ** const srcPtrs = (const void**)malloc(maxNbBlocks * sizeof(void*));
-    size_t* const srcSizes = (size_t* const)malloc(maxNbBlocks * sizeof(size_t));
+    size_t* const srcSizes = (size_t*)malloc(maxNbBlocks * sizeof(size_t));
 
-    void ** const cPtrs = (void** const)malloc(maxNbBlocks * sizeof(void*));
-    size_t* const cSizes = (size_t* const)malloc(maxNbBlocks * sizeof(size_t));
 
-    void ** const resPtrs = (void** const)malloc(maxNbBlocks * sizeof(void*));
+    void ** const cPtrs = (void**)malloc(maxNbBlocks * sizeof(void*));
-    size_t* const resSizes = (size_t* const)malloc(maxNbBlocks * sizeof(size_t));
+    size_t* const cSizes = (size_t*)malloc(maxNbBlocks * sizeof(size_t));
+    size_t* const cCapacities = (size_t*)malloc(maxNbBlocks * sizeof(size_t));
+
+    void ** const resPtrs = (void**)malloc(maxNbBlocks * sizeof(void*));
+    size_t* const resSizes = (size_t*)malloc(maxNbBlocks * sizeof(size_t));
 
-    const size_t maxCompressedSize = ZSTD_compressBound(srcSize) + (maxNbBlocks * 1024);   /* add some room for safety */
-    void* compressedBuffer = malloc(maxCompressedSize);
-    void* resultBuffer = malloc(srcSize);
     BMK_timedFnState_t* timeStateCompress = BMK_createTimeState(adv->nbSeconds);
     BMK_timedFnState_t* timeStateDecompress = BMK_createTimeState(adv->nbSeconds);
 
-    BMK_return_t results;
+    ZSTD_CCtx* ctx = ZSTD_createCCtx();
-    int allocationincomplete = !compressedBuffer || !resultBuffer ||  
+    ZSTD_DCtx* dctx = ZSTD_createDCtx();
-        !srcPtrs || !srcSizes || !cPtrs || !cSizes || !resPtrs || !resSizes;
+
-    if (!allocationincomplete) {
+    const size_t maxCompressedSize = dstCapacity ? dstCapacity : ZSTD_compressBound(srcSize) + (maxNbBlocks * 1024);
-        results = BMK_benchMemAdvancedNoAlloc(srcPtrs, srcSizes, cPtrs, cSizes,
+
-            resPtrs, resSizes, resultBuffer, compressedBuffer, maxCompressedSize, timeStateCompress, timeStateDecompress,
+    void* const internalDstBuffer = dstBuffer ? NULL : malloc(maxCompressedSize);
+    void* const compressedBuffer = dstBuffer ? dstBuffer : internalDstBuffer;
+
+    void* resultBuffer = malloc(srcSize);
+
+    BMK_return_t results = { { 0, 0, 0, 0 }, 0 };
+    int allocationincomplete = !srcPtrs || !srcSizes || !cPtrs || 
+        !cSizes || !cCapacities || !resPtrs || !resSizes || 
+        !timeStateCompress || !timeStateDecompress || !compressedBuffer || !resultBuffer;
+
+    int parametersConflict = !dstBuffer ^ !dstCapacity;
+
+
+    if (!allocationincomplete && !parametersConflict) {
+        results = BMK_benchMemAdvancedNoAlloc(srcPtrs, srcSizes, cPtrs, cCapacities, cSizes,
+            resPtrs, resSizes, &resultBuffer, compressedBuffer, maxCompressedSize, timeStateCompress, timeStateDecompress,
             srcBuffer, srcSize, fileSizes, nbFiles, cLevel, comprParams,
             dictBuffer, dictBufferSize, ctx, dctx, displayLevel, displayName, adv);
     }
+    
     /* clean up */
     BMK_freeTimeState(timeStateCompress);
     BMK_freeTimeState(timeStateDecompress);
-    free(compressedBuffer);
+    
+    ZSTD_freeCCtx(ctx);
+    ZSTD_freeDCtx(dctx);
+
+    free(internalDstBuffer);
     free(resultBuffer);
 
     free((void*)srcPtrs); 
     free(srcSizes); 
     free(cPtrs); 
     free(cSizes);
+    free(cCapacities);
     free(resPtrs);
     free(resSizes);
 
     if(allocationincomplete) {
         EXM_THROW(31, BMK_return_t, "allocation error : not enough memory");
     }
-    results.error = 0;
+    
+    if(parametersConflict) {
+        EXM_THROW(32, BMK_return_t, "Conflicting input results");   
+    }
     return results;
 }
 
@@ -755,42 +786,17 @@ BMK_return_t BMK_benchMem(const void* srcBuffer, size_t srcSize,
                         const size_t* fileSizes, unsigned nbFiles,
                         const int cLevel, const ZSTD_compressionParameters* comprParams,
                         const void* dictBuffer, size_t dictBufferSize,
-                        ZSTD_CCtx* ctx, ZSTD_DCtx* dctx,
                         int displayLevel, const char* displayName) {
 
     const BMK_advancedParams_t adv = BMK_initAdvancedParams();
     return BMK_benchMemAdvanced(srcBuffer, srcSize,
+                                NULL, 0,
                                 fileSizes, nbFiles,
                                 cLevel, comprParams,
                                 dictBuffer, dictBufferSize,
-                                ctx, dctx,
                                 displayLevel, displayName, &adv);
 }
 
-static BMK_return_t BMK_benchMemCtxless(const void* srcBuffer, size_t srcSize,
-                        const size_t* fileSizes, unsigned nbFiles, 
-                        int cLevel, const ZSTD_compressionParameters* const comprParams,
-                        const void* dictBuffer, size_t dictBufferSize,
-                        int displayLevel, const char* displayName, 
-                        const BMK_advancedParams_t* const adv) 
-{
-    BMK_return_t res;
-    ZSTD_CCtx* ctx = ZSTD_createCCtx();
-    ZSTD_DCtx* dctx = ZSTD_createDCtx();
-    if(ctx == NULL || dctx == NULL) {
-        EXM_THROW(12, BMK_return_t, "not enough memory for contexts");
-    }
-    res = BMK_benchMemAdvanced(srcBuffer, srcSize, 
-                fileSizes, nbFiles, 
-                cLevel, comprParams, 
-                dictBuffer, dictBufferSize, 
-                ctx, dctx, 
-                displayLevel, displayName, adv);
-    ZSTD_freeCCtx(ctx);
-    ZSTD_freeDCtx(dctx);
-    return res;
-}
-
 static size_t BMK_findMaxMem(U64 requiredMem)
 {
     size_t const step = 64 MB;
@@ -831,12 +837,12 @@ static BMK_return_t BMK_benchCLevel(const void* srcBuffer, size_t benchedSize,
     if (displayLevel == 1 && !adv->additionalParam)
         DISPLAY("bench %s %s: input %u bytes, %u seconds, %u KB blocks\n", ZSTD_VERSION_STRING, ZSTD_GIT_COMMIT_STRING, (U32)benchedSize, adv->nbSeconds, (U32)(adv->blockSize>>10));
 
-    res = BMK_benchMemCtxless(srcBuffer, benchedSize,
+    res = BMK_benchMemAdvanced(srcBuffer, benchedSize, 
-            fileSizes, nbFiles, 
+                NULL, 0, 
-            cLevel, comprParams, 
+                fileSizes, nbFiles, 
-            dictBuffer, dictBufferSize, 
+                cLevel, comprParams, 
-            displayLevel, displayName, 
+                dictBuffer, dictBufferSize, 
-            adv);
+                displayLevel, displayName, adv);
 
     return res;
 }

programs/bench.h

@@ -32,8 +32,9 @@ extern "C" {
 
 typedef struct {
     size_t cSize;
-    double cSpeed;   /* bytes / sec */
+    U64 cSpeed;   /* bytes / sec */
-    double dSpeed;
+    U64 dSpeed;
+    size_t cMem;
 } BMK_result_t;
 
 ERROR_STRUCT(BMK_result_t, BMK_return_t);
@@ -125,8 +126,6 @@ BMK_return_t BMK_syntheticTest(int cLevel, double compressibility,
  * comprParams - basic compression parameters
  * dictBuffer - a dictionary if used, null otherwise
  * dictBufferSize - size of dictBuffer, 0 otherwise
- * ctx - Compression Context (must be provided)
- * dctx - Decompression Context (must be provided)
  * diplayLevel - see BMK_benchFiles
  * displayName - name used by display
  * return
@@ -138,15 +137,17 @@ BMK_return_t BMK_benchMem(const void* srcBuffer, size_t srcSize,
                         const size_t* fileSizes, unsigned nbFiles,
                         const int cLevel, const ZSTD_compressionParameters* comprParams,
                         const void* dictBuffer, size_t dictBufferSize,
-                        ZSTD_CCtx* ctx, ZSTD_DCtx* dctx,
                         int displayLevel, const char* displayName);
 
-/* See benchMem for normal parameter uses and return, see advancedParams_t for adv */
+/* See benchMem for normal parameter uses and return, see advancedParams_t for adv 
+ * dstBuffer - destination buffer to write compressed output in, NULL if none provided.
+ * dstCapacity - capacity of destination buffer, give 0 if dstBuffer = NULL
+ */
 BMK_return_t BMK_benchMemAdvanced(const void* srcBuffer, size_t srcSize,
+                        void* dstBuffer, size_t dstCapacity, 
                         const size_t* fileSizes, unsigned nbFiles,
                         const int cLevel, const ZSTD_compressionParameters* comprParams,
                         const void* dictBuffer, size_t dictBufferSize,
-                        ZSTD_CCtx* ctx, ZSTD_DCtx* dctx,
                         int displayLevel, const char* displayName,
                         const BMK_advancedParams_t* adv);
 
@@ -170,8 +171,10 @@ typedef size_t (*BMK_initFn_t)(void*);
  * srcBuffers - an array of buffers to be operated on by benchFn
  * srcSizes - an array of the sizes of above buffers
  * dstBuffers - an array of buffers to be written into by benchFn
- * dstCapacities - an array of the capacities of above buffers.
+ * dstCapacities - an array of the capacities of above buffers
+ * blockResults - the return value of benchFn called on each block.
  * nbLoops - defines number of times benchFn is run.
+ * assumed array of size blockCount, will have compressed size of each block written to it.
  * return 
  *      .error will give a nonzero value if ZSTD_isError() is nonzero for any of the return
  *          of the calls to initFn and benchFn, or if benchFunction errors internally
@@ -182,12 +185,11 @@ typedef size_t (*BMK_initFn_t)(void*);
  *          into dstBuffer, hence this value will be the total amount of bytes written to 
  *          dstBuffer.
  */
-BMK_customReturn_t BMK_benchFunction(                        
+BMK_customReturn_t BMK_benchFunction(BMK_benchFn_t benchFn, void* benchPayload,
-                        BMK_benchFn_t benchFn, void* benchPayload,
                         BMK_initFn_t initFn, void* initPayload,
                         size_t blockCount,
                         const void* const * const srcBuffers, const size_t* srcSizes,
-                        void* const * const dstBuffers, const size_t* dstCapacities,
+                        void * const * const dstBuffers, const size_t* dstCapacities, size_t* blockResults,  
                         unsigned nbLoops);
 
 
@@ -216,7 +218,7 @@ BMK_customTimedReturn_t BMK_benchFunctionTimed(BMK_timedFnState_t* cont,
     BMK_initFn_t initFn, void* initPayload,
     size_t blockCount,
     const void* const * const srcBlockBuffers, const size_t* srcBlockSizes,
-    void* const * const dstBlockBuffers, const size_t* dstBlockCapacities);
+    void* const * const dstBlockBuffers, const size_t* dstBlockCapacities, size_t* blockResults);
 
 #endif   /* BENCH_H_121279284357 */
 

tests/README.md

@@ -88,3 +88,34 @@ as well as the 10,000 original files for more detailed comparison of decompressi
 will choose a random seed, and for 1 minute,
 generate random test frames and ensure that the
 zstd library correctly decompresses them in both simple and streaming modes.
+
+#### `paramgrill` - tool for generating compression table parameters and optimizing parameters on file given constraints
+
+Full list of arguments
+```
+ -T#          : set level 1 speed objective
+ -B#          : cut input into blocks of size # (default : single block)
+ -i#          : iteration loops
+ -S           : benchmarks a single run (example command: -Sl3w10h12)
+    w# - windowLog
+    h# - hashLog
+    c# - chainLog
+    s# - searchLog
+    l# - searchLength
+    t# - targetLength
+    S# - strategy
+    L# - level
+ --zstd=      : Single run, parameter selection syntax same as zstdcli
+ --optimize=  : find parameters to maximize compression ratio given parameters
+    Can use all --zstd= commands to constrain the type of solution found in addition to the following constraints
+    cSpeed= - Minimum compression speed
+    dSpeed= - Minimum decompression speed
+    cMem= - compression memory
+    lvl= - Automatically sets compression speed constraint to the speed of that level
+ -P#          : generated sample compressibility 
+ -t#          : Caps runtime of operation in seconds (default : 99999 seconds (about 27 hours )) 
+ -v           : Prints Benchmarking output
+ -D           : Next argument dictionary file
+
+```
+ Any inputs afterwards are treated as files to benchmark.

tests/fullbench.c

@@ -336,7 +336,7 @@ size_t local_ZSTD_decompressContinue(const void* src, size_t srcSize, void* dst,
 static size_t benchMem(const void* src, size_t srcSize, U32 benchNb, int cLevel, ZSTD_compressionParameters* cparams)
 {
     BYTE*  dstBuff;
-    size_t const dstBuffSize = ZSTD_compressBound(srcSize);
+    size_t dstBuffSize = ZSTD_compressBound(srcSize);
     void*  buff2, *buff1;
     const char* benchName;
     BMK_benchFn_t benchFunction;
@@ -514,9 +514,10 @@ static size_t benchMem(const void* src, size_t srcSize, U32 benchNb, int cLevel,
 
     /* benchmark loop */
     {
+        void* dstBuffv = (void*)dstBuff;
         r = BMK_benchFunction(benchFunction, buff2, 
             NULL, NULL,  1, &src, &srcSize, 
-            (void * const * const)&dstBuff, &dstBuffSize, g_nbIterations);
+            &dstBuffv, &dstBuffSize, NULL, g_nbIterations);
         if(r.error) {
             DISPLAY("ERROR %d ! ! \n", r.error);
             errorcode = r.error;

tests/paramgrill.c

@@ -27,7 +27,8 @@
 #include "xxhash.h"
 #include "util.h"
 #include "bench.h"
-
+#include "zstd_errors.h"
+#include "zstd_internal.h"
 
 /*-************************************
 *  Constants
@@ -36,10 +37,7 @@
 #define AUTHOR "Yann Collet"
 #define WELCOME_MESSAGE "*** %s %s %i-bits, by %s ***\n", PROGRAM_DESCRIPTION, ZSTD_VERSION_STRING, (int)(sizeof(void*)*8), AUTHOR
 
-
+#define TIMELOOP_NANOSEC      (1*1000000000ULL) /* 1 second */
-#define KB *(1<<10)
-#define MB *(1<<20)
-#define GB *(1ULL<<30)
 
 #define NBLOOPS    2
 #define TIMELOOP  (2 * SEC_TO_MICRO)
@@ -56,17 +54,54 @@ static const int g_maxNbVariations = 64;
 *  Macros
 **************************************/
 #define DISPLAY(...)  fprintf(stderr, __VA_ARGS__)
+#define TIMED 0
+#ifndef DEBUG
+#  define DEBUG 0
+#endif
+#define DEBUGOUTPUT(...) { if (DEBUG) DISPLAY(__VA_ARGS__); }
 
 #undef MIN
 #undef MAX
 #define MIN(a,b)   ( (a) < (b) ? (a) : (b) )
 #define MAX(a,b)   ( (a) > (b) ? (a) : (b) )
 #define CUSTOM_LEVEL 99
+#define BASE_CLEVEL 1
 
+/* indices for each of the variables */
+typedef enum {
+    wlog_ind = 0,
+    clog_ind = 1,
+    hlog_ind = 2,
+    slog_ind = 3,
+    slen_ind = 4,
+    tlen_ind = 5
+} varInds_t;
+
+#define NUM_PARAMS 6
+/* just don't use strategy as a param. */ 
+
+#undef ZSTD_WINDOWLOG_MAX
+#define ZSTD_WINDOWLOG_MAX 27 //no long range stuff for now. 
+
+#define ZSTD_TARGETLENGTH_MIN 0 
+#define ZSTD_TARGETLENGTH_MAX 999
+
+#define WLOG_RANGE (ZSTD_WINDOWLOG_MAX - ZSTD_WINDOWLOG_MIN + 1)
+#define CLOG_RANGE (ZSTD_CHAINLOG_MAX - ZSTD_CHAINLOG_MIN + 1)
+#define HLOG_RANGE (ZSTD_HASHLOG_MAX - ZSTD_HASHLOG_MIN + 1)
+#define SLOG_RANGE (ZSTD_SEARCHLOG_MAX - ZSTD_SEARCHLOG_MIN + 1)
+#define SLEN_RANGE (ZSTD_SEARCHLENGTH_MAX - ZSTD_SEARCHLENGTH_MIN + 1)
+#define TLEN_RANGE 17
+/* TLEN_RANGE picked manually */
+
+static const int rangetable[NUM_PARAMS] = { WLOG_RANGE, CLOG_RANGE, HLOG_RANGE, SLOG_RANGE, SLEN_RANGE, TLEN_RANGE };
+static const U32 tlen_table[TLEN_RANGE] = { 0, 1, 2, 4, 6, 8, 12, 16, 24, 32, 48, 64, 96, 128, 256, 512, 999 };
 /*-************************************
 *  Benchmark Parameters
 **************************************/
 
+typedef BYTE U8;
+
 static double g_grillDuration_s = 99999;   /* about 27 hours */
 static U32 g_nbIterations = NBLOOPS;
 static double g_compressibility = COMPRESSIBILITY_DEFAULT;
@@ -76,6 +111,7 @@ static U32 g_singleRun = 0;
 static U32 g_target = 0;
 static U32 g_noSeed = 0;
 static ZSTD_compressionParameters g_params = { 0, 0, 0, 0, 0, 0, ZSTD_greedy };
+static UTIL_time_t g_time; /* to be used to compare solution finding speeds to compare to original */
 
 void BMK_SetNbIterations(int nbLoops)
 {
@@ -140,60 +176,504 @@ static unsigned longCommandWArg(const char** stringPtr, const char* longCommand)
     return result;
 }
 
+static U64 g_clockGranularity = 100000000ULL;
+
+static void findClockGranularity(void) {
+    UTIL_time_t clockStart = UTIL_getTime();
+    U64 el1 = 0, el2 = 0;
+    int i = 0;
+    do {
+        el1 = el2;
+        el2 = UTIL_clockSpanNano(clockStart);
+        if(el1 < el2) {
+            U64 iv = el2 - el1;
+            if(g_clockGranularity > iv) {
+                g_clockGranularity = iv;
+                i = 0;
+            } else {
+                i++;
+            }
+        }
+    } while(i < 10);
+    DEBUGOUTPUT("Granularity: %llu\n", (unsigned long long)g_clockGranularity);
+}
+
+typedef struct {
+    U32 cSpeed; /* bytes / sec */
+    U32 dSpeed;
+    U32 cMem;    /* bytes */    
+} constraint_t;
+
+#define CLAMPCHECK(val,min,max) {                     \
+    if (val && (((val)<(min)) | ((val)>(max)))) {     \
+        DISPLAY("INVALID PARAMETER CONSTRAINTS\n");   \
+        return 0;                                     \
+}   }
+
+
+/* Like ZSTD_checkCParams() but allows 0's */
+/* no check on targetLen? */
+static int cParamValid(ZSTD_compressionParameters paramTarget) {
+    CLAMPCHECK(paramTarget.hashLog, ZSTD_HASHLOG_MIN, ZSTD_HASHLOG_MAX);
+    CLAMPCHECK(paramTarget.searchLog, ZSTD_SEARCHLOG_MIN, ZSTD_SEARCHLOG_MAX);
+    CLAMPCHECK(paramTarget.searchLength, ZSTD_SEARCHLENGTH_MIN, ZSTD_SEARCHLENGTH_MAX);
+    CLAMPCHECK(paramTarget.windowLog, ZSTD_WINDOWLOG_MIN, ZSTD_WINDOWLOG_MAX);
+    CLAMPCHECK(paramTarget.chainLog, ZSTD_CHAINLOG_MIN, ZSTD_CHAINLOG_MAX);
+    if(paramTarget.targetLength > ZSTD_TARGETLENGTH_MAX) {
+        DISPLAY("INVALID PARAMETER CONSTRAINTS\n");
+        return 0;
+    }
+    if(paramTarget.strategy > ZSTD_btultra) {
+        DISPLAY("INVALID PARAMETER CONSTRAINTS\n");
+        return 0;
+    }
+    return 1;
+}
+
+static void cParamZeroMin(ZSTD_compressionParameters* paramTarget) {
+    paramTarget->windowLog = paramTarget->windowLog ? paramTarget->windowLog : ZSTD_WINDOWLOG_MIN;
+    paramTarget->searchLog = paramTarget->searchLog ? paramTarget->searchLog : ZSTD_SEARCHLOG_MIN;
+    paramTarget->chainLog = paramTarget->chainLog ? paramTarget->chainLog : ZSTD_CHAINLOG_MIN;
+    paramTarget->hashLog = paramTarget->hashLog ? paramTarget->hashLog : ZSTD_HASHLOG_MIN;
+    paramTarget->searchLength = paramTarget->searchLength ? paramTarget->searchLength : ZSTD_SEARCHLENGTH_MIN;
+    paramTarget->targetLength = paramTarget->targetLength ? paramTarget->targetLength : 0;
+}
+
+static void BMK_translateAdvancedParams(const ZSTD_compressionParameters params)
+{
+    DISPLAY("--zstd=windowLog=%u,chainLog=%u,hashLog=%u,searchLog=%u,searchLength=%u,targetLength=%u,strategy=%u \n",
+             params.windowLog, params.chainLog, params.hashLog, params.searchLog, params.searchLength, params.targetLength, (U32)(params.strategy));
+}
+
+/* checks results are feasible */
+static int feasible(const BMK_result_t results, const constraint_t target) {
+    return (results.cSpeed >= target.cSpeed) && (results.dSpeed >= target.dSpeed) && (results.cMem <= target.cMem);
+}
+
+/* hill climbing value for part 1 */
+/* Scoring here is a linear reward for all set constraints normalized between 0 to 1 
+ * (with 0 at 0 and 1 being fully fulfilling the constraint), summed with a logarithmic 
+ * bonus to exceeding the constraint value. We also give linear ratio for compression ratio.
+ * The constant factors are experimental. 
+ */
+static double resultScore(const BMK_result_t res, const size_t srcSize, const constraint_t target) {
+    double cs = 0., ds = 0., rt, cm = 0.;
+    const double r1 = 1, r2 = 0.1, rtr = 0.5;
+    double ret;
+    if(target.cSpeed) { cs = res.cSpeed / (double)target.cSpeed; }
+    if(target.dSpeed) { ds = res.dSpeed / (double)target.dSpeed; }
+    if(target.cMem != (U32)-1) { cm = (double)target.cMem / res.cMem; }
+    rt = ((double)srcSize / res.cSize);
+
+    ret = (MIN(1, cs) + MIN(1, ds)  + MIN(1, cm))*r1 + rt * rtr + 
+         (MAX(0, log(cs))+ MAX(0, log(ds))+ MAX(0, log(cm))) * r2;
+    return ret;
+}
+
+/* return true if r2 strictly better than r1 */ 
+static int compareResultLT(const BMK_result_t result1, const BMK_result_t result2, const constraint_t target, size_t srcSize) {
+    if(feasible(result1, target) && feasible(result2, target)) {
+        return (result1.cSize > result2.cSize) || (result1.cSize == result2.cSize && result2.cSpeed > result1.cSpeed)
+        || (result1.cSize == result2.cSize && result2.cSpeed == result1.cSpeed && result2.dSpeed > result1.dSpeed);
+    }
+    return feasible(result2, target) || (!feasible(result1, target) && (resultScore(result1, srcSize, target) < resultScore(result2, srcSize, target)));
+
+}
+
+/* factor sort of arbitrary */
+static constraint_t relaxTarget(constraint_t target) {
+    target.cMem = (U32)-1;
+    target.cSpeed *= 0.9; 
+    target.dSpeed *= 0.9;
+    return target;
+}
+
 /*-*******************************************************
 *  Bench functions
 *********************************************************/
 
-typedef struct
-{
-    const char* srcPtr;
-    size_t srcSize;
-    char*  cPtr;
-    size_t cRoom;
-    size_t cSize;
-    char*  resPtr;
-    size_t resSize;
-} blockParam_t;
-
-
 const char* g_stratName[ZSTD_btultra+1] = {
                 "(none)       ", "ZSTD_fast    ", "ZSTD_dfast   ",
                 "ZSTD_greedy  ", "ZSTD_lazy    ", "ZSTD_lazy2   ",
                 "ZSTD_btlazy2 ", "ZSTD_btopt   ", "ZSTD_btultra "};
 
-/* TODO: support additional parameters (more files, fileSizes) */
+/* benchParam but only takes in one input buffer. */
-static size_t
+static int
-BMK_benchParam(BMK_result_t* resultPtr,
+BMK_benchParam1(BMK_result_t* resultPtr,
                const void* srcBuffer, size_t srcSize,
-               ZSTD_CCtx* ctx, ZSTD_DCtx* dctx, 
                const ZSTD_compressionParameters cParams) {
 
-    BMK_return_t res = BMK_benchMem(srcBuffer,srcSize, &srcSize, 1, 0, &cParams, NULL, 0, ctx, dctx, 0, "File");
+    BMK_return_t res = BMK_benchMem(srcBuffer,srcSize, &srcSize, 1, BASE_CLEVEL, &cParams, NULL, 0, 0, "File");
     *resultPtr = res.result;
     return res.error;
 }
 
-static void BMK_printWinner(FILE* f, U32 cLevel, BMK_result_t result, ZSTD_compressionParameters params, size_t srcSize)
-{
-    char lvlstr[15] = "Custom Level";
-    DISPLAY("\r%79s\r", "");
-    fprintf(f,"    {%3u,%3u,%3u,%3u,%3u,%3u, %s },  ",
-            params.windowLog, params.chainLog, params.hashLog, params.searchLog, params.searchLength,
-            params.targetLength, g_stratName[(U32)(params.strategy)]);
-    if(cLevel != CUSTOM_LEVEL) {
-        snprintf(lvlstr, 15, "  Level %2u  ", cLevel);
-    }
-    fprintf(f,
-        "/* %s */   /* R:%5.3f at %5.1f MB/s - %5.1f MB/s */\n",
-        lvlstr, (double)srcSize / result.cSize, result.cSpeed / 1000000., result.dSpeed / 1000000.);
-}
-
-
 typedef struct {
     BMK_result_t result;
     ZSTD_compressionParameters params;
 } winnerInfo_t;
 
+static ZSTD_compressionParameters emptyParams(void) {
+    ZSTD_compressionParameters p = { 0, 0, 0, 0, 0, 0, (ZSTD_strategy)0 };
+    return p;
+}
+
+static winnerInfo_t initWinnerInfo(ZSTD_compressionParameters p) {
+    winnerInfo_t w1;
+    w1.result.cSpeed = 0.;
+    w1.result.dSpeed = 0.;
+    w1.result.cMem = (size_t)-1;
+    w1.result.cSize = (size_t)-1;
+    w1.params = p;
+    return w1;
+}
+
+typedef struct {
+    void* srcBuffer;
+    size_t srcSize;
+    const void** srcPtrs;
+    size_t* srcSizes;
+    void** dstPtrs;
+    size_t* dstCapacities;
+    size_t* dstSizes;
+    void** resPtrs;
+    size_t* resSizes;
+    size_t nbBlocks;
+} buffers_t;
+
+typedef struct {
+    size_t dictSize;
+    void* dictBuffer;
+    ZSTD_CCtx* cctx;
+    ZSTD_DCtx* dctx;
+} contexts_t;
+
+static int
+BMK_benchParam(BMK_result_t* resultPtr,
+                const buffers_t buf, const contexts_t ctx,
+                const ZSTD_compressionParameters cParams) {
+    BMK_return_t res = BMK_benchMem(buf.srcPtrs[0], buf.srcSize, buf.srcSizes, (unsigned)buf.nbBlocks, 0, &cParams, ctx.dictBuffer, ctx.dictSize, 0, "Files");
+    *resultPtr = res.result;
+    return res.error;
+}
+
+/*-*******************************************************
+*  From Paramgrill
+*********************************************************/
+
+static void BMK_initCCtx(ZSTD_CCtx* ctx, 
+    const void* dictBuffer, const size_t dictBufferSize, const int cLevel, 
+    const ZSTD_compressionParameters* comprParams, const BMK_advancedParams_t* adv) {
+    ZSTD_CCtx_reset(ctx);
+    ZSTD_CCtx_resetParameters(ctx);
+    if (adv->nbWorkers==1) {
+        ZSTD_CCtx_setParameter(ctx, ZSTD_p_nbWorkers, 0);
+    } else {
+        ZSTD_CCtx_setParameter(ctx, ZSTD_p_nbWorkers, adv->nbWorkers);
+    }
+    ZSTD_CCtx_setParameter(ctx, ZSTD_p_compressionLevel, cLevel);
+    ZSTD_CCtx_setParameter(ctx, ZSTD_p_enableLongDistanceMatching, adv->ldmFlag);
+    ZSTD_CCtx_setParameter(ctx, ZSTD_p_ldmMinMatch, adv->ldmMinMatch);
+    ZSTD_CCtx_setParameter(ctx, ZSTD_p_ldmHashLog, adv->ldmHashLog);
+    ZSTD_CCtx_setParameter(ctx, ZSTD_p_ldmBucketSizeLog, adv->ldmBucketSizeLog);
+    ZSTD_CCtx_setParameter(ctx, ZSTD_p_ldmHashEveryLog, adv->ldmHashEveryLog);
+    ZSTD_CCtx_setParameter(ctx, ZSTD_p_windowLog, comprParams->windowLog);
+    ZSTD_CCtx_setParameter(ctx, ZSTD_p_hashLog, comprParams->hashLog);
+    ZSTD_CCtx_setParameter(ctx, ZSTD_p_chainLog, comprParams->chainLog);
+    ZSTD_CCtx_setParameter(ctx, ZSTD_p_searchLog, comprParams->searchLog);
+    ZSTD_CCtx_setParameter(ctx, ZSTD_p_minMatch, comprParams->searchLength);
+    ZSTD_CCtx_setParameter(ctx, ZSTD_p_targetLength, comprParams->targetLength);
+    ZSTD_CCtx_setParameter(ctx, ZSTD_p_compressionStrategy, comprParams->strategy);
+    ZSTD_CCtx_loadDictionary(ctx, dictBuffer, dictBufferSize);
+}
+
+
+static void BMK_initDCtx(ZSTD_DCtx* dctx, 
+    const void* dictBuffer, const size_t dictBufferSize) {
+    ZSTD_DCtx_reset(dctx);
+    ZSTD_DCtx_loadDictionary(dctx, dictBuffer, dictBufferSize);
+}
+
+typedef struct {
+    ZSTD_CCtx* ctx;
+    const void* dictBuffer;
+    size_t dictBufferSize;
+    int cLevel;
+    const ZSTD_compressionParameters* comprParams;
+    const BMK_advancedParams_t* adv;
+} BMK_initCCtxArgs;
+
+static size_t local_initCCtx(void* payload) {
+    const BMK_initCCtxArgs* ag = (const BMK_initCCtxArgs*)payload;
+    BMK_initCCtx(ag->ctx, ag->dictBuffer, ag->dictBufferSize, ag->cLevel, ag->comprParams, ag->adv);
+    return 0;
+}
+
+typedef struct {
+    ZSTD_DCtx* dctx;
+    const void* dictBuffer;
+    size_t dictBufferSize;
+} BMK_initDCtxArgs;
+
+static size_t local_initDCtx(void* payload) {
+    const BMK_initDCtxArgs* ag = (const BMK_initDCtxArgs*)payload;
+    BMK_initDCtx(ag->dctx, ag->dictBuffer, ag->dictBufferSize);
+    return 0;
+}
+
+/* additional argument is just the context */
+static size_t local_defaultCompress(
+    const void* srcBuffer, size_t srcSize, 
+    void* dstBuffer, size_t dstSize, 
+    void* addArgs) {
+    size_t moreToFlush = 1;
+    ZSTD_CCtx* ctx = (ZSTD_CCtx*)addArgs;
+    ZSTD_inBuffer in;
+    ZSTD_outBuffer out;
+    in.src = srcBuffer;
+    in.size = srcSize;
+    in.pos = 0;
+    out.dst = dstBuffer;
+    out.size = dstSize;
+    out.pos = 0;
+    assert(dstSize == ZSTD_compressBound(srcSize)); /* specific to this version, which is only used in paramgrill */
+    while (moreToFlush) {
+        if(out.pos == out.size) {
+            return (size_t)-ZSTD_error_dstSize_tooSmall;
+        }
+        moreToFlush = ZSTD_compress_generic(ctx, &out, &in, ZSTD_e_end);
+        if (ZSTD_isError(moreToFlush)) {
+            return moreToFlush;
+        }
+    }
+    return out.pos;
+}
+
+/* additional argument is just the context */
+static size_t local_defaultDecompress(
+    const void* srcBuffer, size_t srcSize, 
+    void* dstBuffer, size_t dstSize, 
+    void* addArgs) {
+    size_t moreToFlush = 1;
+    ZSTD_DCtx* dctx = (ZSTD_DCtx*)addArgs;
+    ZSTD_inBuffer in;
+    ZSTD_outBuffer out;
+    in.src = srcBuffer;
+    in.size = srcSize;
+    in.pos = 0;
+    out.dst = dstBuffer;
+    out.size = dstSize;
+    out.pos = 0;
+    while (moreToFlush) {
+        if(out.pos == out.size) {
+            return (size_t)-ZSTD_error_dstSize_tooSmall;
+        }
+        moreToFlush = ZSTD_decompress_generic(dctx,
+                            &out, &in);
+        if (ZSTD_isError(moreToFlush)) {
+            return moreToFlush;
+        }
+    }
+    return out.pos;
+
+}
+
+/*-*******************************************************
+*  From Paramgrill End
+*********************************************************/
+
+/* Replicate functionality of benchMemAdvanced, but with pre-split src / dst buffers */
+/* The purpose is so that sufficient information is returned so that a decompression call to benchMemInvertible is possible */
+/* BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, fileSizes, nbFiles, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); */
+/* nbSeconds used in same way as in BMK_advancedParams_t, as nbIters when in iterMode */
+
+/* if in decodeOnly, then srcPtr's will be compressed blocks, and uncompressedBlocks will be written to dstPtrs? */
+/* dictionary nullable, nothing else though. */
+static BMK_return_t BMK_benchMemInvertible(const buffers_t buf, const contexts_t ctx, 
+                        const int cLevel, const ZSTD_compressionParameters* comprParams,
+                        const BMK_mode_t mode, const BMK_loopMode_t loopMode, const unsigned nbSeconds) {
+
+    U32 i;
+    BMK_return_t results = { { 0, 0., 0., 0 }, 0 } ;
+    const void *const *const srcPtrs = (const void *const *const)buf.srcPtrs;
+    size_t const *const srcSizes = buf.srcSizes;
+    void** const dstPtrs = buf.dstPtrs;
+    size_t const *const dstCapacities = buf.dstCapacities;
+    size_t* const dstSizes = buf.dstSizes;
+    void** const resPtrs = buf.resPtrs;
+    size_t const *const resSizes = buf.resSizes;
+    const void* dictBuffer = ctx.dictBuffer;
+    const size_t dictBufferSize = ctx.dictSize;
+    const size_t nbBlocks = buf.nbBlocks;
+    const size_t srcSize = buf.srcSize;
+    ZSTD_CCtx* cctx = ctx.cctx;
+    ZSTD_DCtx* dctx = ctx.dctx;
+
+    BMK_advancedParams_t adv = BMK_initAdvancedParams();
+    adv.mode = mode;
+    adv.loopMode = loopMode;
+    adv.nbSeconds = nbSeconds;
+
+    /* warmimg up memory */
+    /* can't do this if decode only */
+    for(i = 0; i < buf.nbBlocks; i++) {
+        if(mode != BMK_decodeOnly) {
+            RDG_genBuffer(dstPtrs[i], dstCapacities[i], 0.10, 0.50, 1);
+        } else {
+            RDG_genBuffer(resPtrs[i], resSizes[i], 0.10, 0.50, 1);
+        }
+    }
+
+    /* Bench */
+       
+    {
+        /* init args */
+        BMK_initCCtxArgs cctxprep;
+        BMK_initDCtxArgs dctxprep;
+        cctxprep.ctx = cctx;
+        cctxprep.dictBuffer = dictBuffer;
+        cctxprep.dictBufferSize = dictBufferSize;
+        cctxprep.cLevel = cLevel;
+        cctxprep.comprParams = comprParams;
+        cctxprep.adv = &adv;
+        dctxprep.dctx = dctx;
+        dctxprep.dictBuffer = dictBuffer;
+        dctxprep.dictBufferSize = dictBufferSize;
+
+        if(loopMode == BMK_timeMode) {
+            BMK_customTimedReturn_t intermediateResultCompress;
+            BMK_customTimedReturn_t intermediateResultDecompress;  
+            BMK_timedFnState_t* timeStateCompress = BMK_createTimeState(nbSeconds);
+            BMK_timedFnState_t* timeStateDecompress = BMK_createTimeState(nbSeconds);
+            if(mode == BMK_compressOnly) {
+                intermediateResultCompress.completed = 0;
+                intermediateResultDecompress.completed = 1;
+            } else if (mode == BMK_decodeOnly) {
+                intermediateResultCompress.completed = 1;
+                intermediateResultDecompress.completed = 0;
+            } else { /* both */
+                intermediateResultCompress.completed = 0;
+                intermediateResultDecompress.completed = 0;
+            }
+
+            while(!intermediateResultCompress.completed) {
+                intermediateResultCompress = BMK_benchFunctionTimed(timeStateCompress, &local_defaultCompress, (void*)cctx, &local_initCCtx, (void*)&cctxprep,
+                    nbBlocks, srcPtrs, srcSizes, dstPtrs, dstCapacities, dstSizes);
+
+                if(intermediateResultCompress.result.error) {
+                    results.error = intermediateResultCompress.result.error;
+                    BMK_freeTimeState(timeStateCompress);
+                    BMK_freeTimeState(timeStateDecompress);
+                    return results;
+                }
+                results.result.cSpeed = (srcSize * TIMELOOP_NANOSEC) / intermediateResultCompress.result.result.nanoSecPerRun;
+                results.result.cSize = intermediateResultCompress.result.result.sumOfReturn;
+            }
+
+            while(!intermediateResultDecompress.completed) {
+                intermediateResultDecompress = BMK_benchFunctionTimed(timeStateDecompress, &local_defaultDecompress, (void*)(dctx), &local_initDCtx, (void*)&dctxprep,
+                    nbBlocks, (const void* const*)dstPtrs, dstSizes, resPtrs, resSizes, NULL);
+
+                if(intermediateResultDecompress.result.error) {
+                    results.error = intermediateResultDecompress.result.error;
+                    BMK_freeTimeState(timeStateCompress);
+                    BMK_freeTimeState(timeStateDecompress);
+                    return results;
+                }
+                results.result.dSpeed = (srcSize * TIMELOOP_NANOSEC) / intermediateResultDecompress.result.result.nanoSecPerRun;
+            }
+
+            BMK_freeTimeState(timeStateCompress);
+            BMK_freeTimeState(timeStateDecompress);
+
+        } else { /* iterMode; */
+            if(mode != BMK_decodeOnly) {
+
+                BMK_customReturn_t compressionResults = BMK_benchFunction(&local_defaultCompress, (void*)cctx, &local_initCCtx, (void*)&cctxprep,
+                    nbBlocks, srcPtrs, srcSizes, dstPtrs, dstCapacities, dstSizes, nbSeconds); 
+                if(compressionResults.error) {
+                    results.error = compressionResults.error;
+                    return results;
+                }
+                if(compressionResults.result.nanoSecPerRun == 0) {
+                    results.result.cSpeed = 0;
+                } else {
+                    results.result.cSpeed = srcSize * TIMELOOP_NANOSEC / compressionResults.result.nanoSecPerRun;
+                }
+                results.result.cSize = compressionResults.result.sumOfReturn;
+            }
+
+            if(mode != BMK_compressOnly) {
+                BMK_customReturn_t decompressionResults;
+                decompressionResults = BMK_benchFunction(
+                    &local_defaultDecompress, (void*)(dctx),
+                    &local_initDCtx, (void*)&dctxprep, nbBlocks,
+                    (const void* const*)dstPtrs, dstSizes, resPtrs, resSizes, NULL,
+                    nbSeconds);
+
+                if(decompressionResults.error) {
+                    results.error = decompressionResults.error;
+                    return results;
+                }
+
+                if(decompressionResults.result.nanoSecPerRun == 0) {
+                    results.result.dSpeed = 0;
+                } else {
+                    results.result.dSpeed = srcSize * TIMELOOP_NANOSEC / decompressionResults.result.nanoSecPerRun;
+                }
+            }
+        }
+    }
+   /* Bench */
+    results.result.cMem = (1 << (comprParams->windowLog)) + ZSTD_sizeof_CCtx(cctx);
+    return results;
+}
+
+
+static void BMK_printWinner(FILE* f, const U32 cLevel, const BMK_result_t result, const ZSTD_compressionParameters params, const size_t srcSize)
+{
+    char lvlstr[15] = "Custom Level";
+    const U64 time = UTIL_clockSpanNano(g_time);
+    const U64 minutes = time / (60ULL * TIMELOOP_NANOSEC);
+
+    DISPLAY("\r%79s\r", "");
+
+    fprintf(f,"    {%3u,%3u,%3u,%3u,%3u,%3u, %s },  ",
+        params.windowLog, params.chainLog, params.hashLog, params.searchLog, params.searchLength,
+        params.targetLength, g_stratName[(U32)(params.strategy)]);
+
+    if(cLevel != CUSTOM_LEVEL) {
+        snprintf(lvlstr, 15, "  Level %2u  ", cLevel);
+    }
+
+    fprintf(f,
+        "/* %s */   /* R:%5.3f at %5.1f MB/s - %5.1f MB/s */",
+        lvlstr, (double)srcSize / result.cSize, (double)result.cSpeed / (1 << 20), (double)result.dSpeed / (1 << 20));
+
+    if(TIMED) { fprintf(f, " - %1lu:%2lu:%05.2f", (unsigned long) minutes / 60,(unsigned long) minutes % 60, (double)(time - minutes * TIMELOOP_NANOSEC * 60ULL)/TIMELOOP_NANOSEC); }
+    fprintf(f, "\n"); 
+}
+
+static void BMK_printWinnerOpt(FILE* f, const U32 cLevel, const BMK_result_t result, const ZSTD_compressionParameters params, const constraint_t targetConstraints, const size_t srcSize)
+{
+    /* global winner used for constraints */
+    static winnerInfo_t g_winner = { { 0, 0, (size_t)-1, (size_t)-1 } , { 0, 0, 0, 0, 0, 0, ZSTD_fast } }; 
+    
+    if(DEBUG || compareResultLT(g_winner.result, result, targetConstraints, srcSize)) {
+        if(DEBUG && compareResultLT(g_winner.result, result, targetConstraints, srcSize)) {
+            DISPLAY("New Winner: \n");
+        }
+
+        BMK_printWinner(f, cLevel, result, params, srcSize);
+
+        if(compareResultLT(g_winner.result, result, targetConstraints, srcSize)) {
+            BMK_translateAdvancedParams(params);
+            g_winner.result = result;
+            g_winner.params = params;
+        }
+    }  
+}
+
 static void BMK_printWinners2(FILE* f, const winnerInfo_t* winners, size_t srcSize)
 {
     int cLevel;
@@ -216,8 +696,8 @@ static void BMK_printWinners(FILE* f, const winnerInfo_t* winners, size_t srcSiz
 
 
 typedef struct {
-    double cSpeed_min;
+    U64 cSpeed_min;
-    double dSpeed_min;
+    U64 dSpeed_min;
     U32 windowLog_max;
     ZSTD_strategy strategy_max;
 } level_constraints_t;
@@ -244,14 +724,13 @@ static void BMK_init_level_constraints(int bytePerSec_level1)
 }
 
 static int BMK_seed(winnerInfo_t* winners, const ZSTD_compressionParameters params,
-              const void* srcBuffer, size_t srcSize,
+              const void* srcBuffer, size_t srcSize)
-                    ZSTD_CCtx* ctx, ZSTD_DCtx* dctx)
 {
     BMK_result_t testResult;
     int better = 0;
     int cLevel;
 
-    BMK_benchParam(&testResult, srcBuffer, srcSize, ctx, dctx, params);
+    BMK_benchParam1(&testResult, srcBuffer, srcSize, params);
 
 
     for (cLevel = 1; cLevel <= NB_LEVELS_TRACKED; cLevel++) {
@@ -314,16 +793,16 @@ static int BMK_seed(winnerInfo_t* winners, const ZSTD_compressionParameters para
                 /* too large compression speed difference for the compression benefit */
                 if (W_ratio > O_ratio)
                 DISPLAY ("Compression Speed : %5.3f @ %4.1f MB/s  vs  %5.3f @ %4.1f MB/s   : not enough for level %i\n",
-                         W_ratio, testResult.cSpeed / 1000000,
+                         W_ratio, (double)testResult.cSpeed / (1 MB),
-                         O_ratio, winners[cLevel].result.cSpeed / 1000000.,   cLevel);
+                         O_ratio, (double)winners[cLevel].result.cSpeed / (1 MB),   cLevel);
                 continue;
             }
             if (W_DSpeed_note   < O_DSpeed_note  ) {
                 /* too large decompression speed difference for the compression benefit */
                 if (W_ratio > O_ratio)
                 DISPLAY ("Decompression Speed : %5.3f @ %4.1f MB/s  vs  %5.3f @ %4.1f MB/s   : not enough for level %i\n",
-                         W_ratio, testResult.dSpeed / 1000000.,
+                         W_ratio, (double)testResult.dSpeed / (1 MB),
-                         O_ratio, winners[cLevel].result.dSpeed / 1000000.,   cLevel);
+                         O_ratio, (double)winners[cLevel].result.dSpeed / (1 MB),   cLevel);
                 continue;
             }
 
@@ -340,152 +819,386 @@ static int BMK_seed(winnerInfo_t* winners, const ZSTD_compressionParameters para
     return better;
 }
 
-
+/* bounds check in sanitize too? */
-/* nullified useless params, to ensure count stats */
+#define CLAMP(var, lo, hi) {                          \
-static ZSTD_compressionParameters* sanitizeParams(ZSTD_compressionParameters params)
+    var = MAX(MIN(var, hi), lo);                      \
-{
-    g_params = params;
-    if (params.strategy == ZSTD_fast)
-        g_params.chainLog = 0, g_params.searchLog = 0;
-    if (params.strategy == ZSTD_dfast)
-        g_params.searchLog = 0;
-    if (params.strategy != ZSTD_btopt && params.strategy != ZSTD_btultra)
-        g_params.targetLength = 0;
-    return &g_params;
 }
 
+/* nullified useless params, to ensure count stats */
+/* no point in windowLog < chainLog (no point 2x chainLog for bt) */
+/* now with built in bounds-checking */
+/* no longer does anything with sanitizeVarArray + clampcheck */
+static ZSTD_compressionParameters sanitizeParams(ZSTD_compressionParameters params)
+{
+    if (params.strategy == ZSTD_fast)
+        params.chainLog = 0, params.searchLog = 0;
+    if (params.strategy == ZSTD_dfast)
+        params.searchLog = 0;
+    if (params.strategy != ZSTD_btopt && params.strategy != ZSTD_btultra && params.strategy != ZSTD_fast)
+        params.targetLength = 0;
 
-static void paramVariation(ZSTD_compressionParameters* ptr)
+    return params;
+}
+
+/* new length */
+/* keep old array, will need if iter over strategy. */
+static int sanitizeVarArray(varInds_t* varNew, const int varLength, const varInds_t* varArray, const ZSTD_strategy strat) {
+    int i, j = 0;
+    for(i = 0; i < varLength; i++) {
+        if( !((varArray[i] == clog_ind && strat == ZSTD_fast)
+            || (varArray[i] == slog_ind && strat == ZSTD_dfast) 
+            || (varArray[i] == tlen_ind && strat != ZSTD_btopt && strat != ZSTD_btultra && strat != ZSTD_fast))) {
+            varNew[j] = varArray[i];
+            j++;
+        }
+    }
+    return j;
+
+}
+
+/* res should be NUM_PARAMS size */
+/* constructs varArray from ZSTD_compressionParameters style parameter */
+static int variableParams(const ZSTD_compressionParameters paramConstraints, varInds_t* res) {
+    int j = 0;
+    if(!paramConstraints.windowLog) {
+        res[j] = wlog_ind;
+        j++;
+    }
+    if(!paramConstraints.chainLog) {
+        res[j] = clog_ind;
+        j++;
+    }
+    if(!paramConstraints.hashLog) {
+        res[j] = hlog_ind;
+        j++;
+    }
+    if(!paramConstraints.searchLog) {
+        res[j] = slog_ind;
+        j++;
+    }
+    if(!paramConstraints.searchLength) {
+        res[j] = slen_ind;
+        j++;
+    }
+    if(!paramConstraints.targetLength) {
+        res[j] = tlen_ind;
+        j++;
+    }
+    return j;
+}
+
+/* bin-search on tlen_table for correct index. */
+static int tlen_inv(U32 x) {
+    int lo = 0;
+    int hi = TLEN_RANGE;
+    while(lo < hi) {
+        int mid = (lo + hi) / 2;
+        if(tlen_table[mid] < x) {
+            lo = mid + 1;
+        } if(tlen_table[mid] == x) {
+            return mid;
+        } else {
+            hi = mid;
+        }
+    }
+    return lo;
+}
+
+/* amt will probably always be \pm 1? */
+/* slight change from old paramVariation, targetLength can only take on powers of 2 now (999 ~= 1024?) */
+/* take max/min bounds into account as well? */
+static void paramVaryOnce(const varInds_t paramIndex, const int amt, ZSTD_compressionParameters* ptr) {
+    switch(paramIndex)
+    {
+        case wlog_ind: ptr->windowLog    += amt; break;
+        case clog_ind: ptr->chainLog     += amt; break;
+        case hlog_ind: ptr->hashLog      += amt; break;
+        case slog_ind: ptr->searchLog    += amt; break;
+        case slen_ind: ptr->searchLength += amt; break;
+        case tlen_ind: 
+            ptr->targetLength = tlen_table[MAX(0, MIN(TLEN_RANGE - 1, tlen_inv(ptr->targetLength) + amt))];
+            break;
+        default: break;
+    }
+}
+
+/* varies ptr by nbChanges respecting varyParams*/
+static void paramVariation(ZSTD_compressionParameters* ptr, const varInds_t* varyParams, const int varyLen, const U32 nbChanges)
 {
     ZSTD_compressionParameters p;
     U32 validated = 0;
     while (!validated) {
-        U32 nbChanges = (FUZ_rand(&g_rand) & 3) + 1;
+        U32 i;
         p = *ptr;
-        for ( ; nbChanges ; nbChanges--) {
+        for (i = 0 ; i < nbChanges ; i++) {
-            const U32 changeID = FUZ_rand(&g_rand) % 14;
+            const U32 changeID = FUZ_rand(&g_rand) % (varyLen << 1);
-            switch(changeID)
+            paramVaryOnce(varyParams[changeID >> 1], ((changeID & 1) << 1) - 1, &p);
-            {
-            case 0:
-                p.chainLog++; break;
-            case 1:
-                p.chainLog--; break;
-            case 2:
-                p.hashLog++; break;
-            case 3:
-                p.hashLog--; break;
-            case 4:
-                p.searchLog++; break;
-            case 5:
-                p.searchLog--; break;
-            case 6:
-                p.windowLog++; break;
-            case 7:
-                p.windowLog--; break;
-            case 8:
-                p.searchLength++; break;
-            case 9:
-                p.searchLength--; break;
-            case 10:
-                p.strategy = (ZSTD_strategy)(((U32)p.strategy)+1); break;
-            case 11:
-                p.strategy = (ZSTD_strategy)(((U32)p.strategy)-1); break;
-            case 12:
-                p.targetLength *= 1 + ((double)(FUZ_rand(&g_rand)&255)) / 256.; break;
-            case 13:
-                p.targetLength /= 1 + ((double)(FUZ_rand(&g_rand)&255)) / 256.; break;
-            }
         }
         validated = !ZSTD_isError(ZSTD_checkCParams(p));
     }
     *ptr = p;
 }
 
+/* length of memo table given free variables */
+static size_t memoTableLen(const varInds_t* varyParams, const int varyLen) {
+    size_t arrayLen = 1;
+    int i;
+    for(i = 0; i < varyLen; i++) {
+        arrayLen *= rangetable[varyParams[i]];
+    }
+    return arrayLen;
+}
+
+/* returns unique index in memotable of compression parameters */
+static unsigned memoTableInd(const ZSTD_compressionParameters* ptr, const varInds_t* varyParams, const int varyLen) {
+    int i;
+    unsigned ind = 0;
+    for(i = 0; i < varyLen; i++) {
+        switch(varyParams[i]) {
+            case wlog_ind: ind *= WLOG_RANGE; ind += ptr->windowLog              
+                - ZSTD_WINDOWLOG_MIN   ; break;
+            case clog_ind: ind *= CLOG_RANGE; ind += ptr->chainLog               
+                - ZSTD_CHAINLOG_MIN    ; break;
+            case hlog_ind: ind *= HLOG_RANGE; ind += ptr->hashLog                
+                - ZSTD_HASHLOG_MIN     ; break;
+            case slog_ind: ind *= SLOG_RANGE; ind += ptr->searchLog              
+                - ZSTD_SEARCHLOG_MIN   ; break;
+            case slen_ind: ind *= SLEN_RANGE; ind += ptr->searchLength           
+                - ZSTD_SEARCHLENGTH_MIN; break;
+            case tlen_ind: ind *= TLEN_RANGE; ind += tlen_inv(ptr->targetLength) 
+                - ZSTD_TARGETLENGTH_MIN; break;
+        }
+    }
+    return ind;
+}
+
+/* inverse of above function (from index to parameters) */
+static void memoTableIndInv(ZSTD_compressionParameters* ptr, const varInds_t* varyParams, const int varyLen, size_t ind) {
+    int i;
+    for(i = varyLen - 1; i >= 0; i--) {
+        switch(varyParams[i]) {
+            case wlog_ind: ptr->windowLog    = ind % WLOG_RANGE + ZSTD_WINDOWLOG_MIN;
+                ind /= WLOG_RANGE; break;
+            case clog_ind: ptr->chainLog     = ind % CLOG_RANGE + ZSTD_CHAINLOG_MIN;
+                ind /= CLOG_RANGE; break;
+            case hlog_ind: ptr->hashLog      = ind % HLOG_RANGE + ZSTD_HASHLOG_MIN;
+                ind /= HLOG_RANGE; break;
+            case slog_ind: ptr->searchLog    = ind % SLOG_RANGE + ZSTD_SEARCHLOG_MIN;
+                ind /= SLOG_RANGE; break;
+            case slen_ind: ptr->searchLength = ind % SLEN_RANGE + ZSTD_SEARCHLENGTH_MIN;
+                ind /= SLEN_RANGE; break;
+            case tlen_ind: ptr->targetLength = tlen_table[(ind % TLEN_RANGE)];
+                ind /= TLEN_RANGE; break;
+        }
+    }
+}
+
+/* Initialize memoization table, which tracks and prevents repeated benchmarking
+ * of the same set of parameters. In addition, it is also used to immediately mark 
+ * redundant / obviously non-optimal parameter configurations (e.g. wlog - 1 larger)
+ * than srcSize, clog > wlog, ... 
+ */
+static void initMemoTable(U8* memoTable, ZSTD_compressionParameters paramConstraints, const constraint_t target, const varInds_t* varyParams, const int varyLen, const size_t srcSize) {
+    size_t i;
+    size_t arrayLen = memoTableLen(varyParams, varyLen);
+    int cwFixed = !paramConstraints.chainLog || !paramConstraints.windowLog;
+    int scFixed = !paramConstraints.searchLog || !paramConstraints.chainLog;
+    int whFixed = !paramConstraints.windowLog || !paramConstraints.hashLog;
+    int wFixed = !paramConstraints.windowLog;
+    int j = 0;
+    assert(memoTable != NULL);
+    memset(memoTable, 0, arrayLen);
+    cParamZeroMin(&paramConstraints);
+
+    for(i = 0; i < arrayLen; i++) {
+        memoTableIndInv(&paramConstraints, varyParams, varyLen, i);
+        if(ZSTD_estimateCStreamSize_usingCParams(paramConstraints) > (size_t)target.cMem) {
+            memoTable[i] = 255;
+            j++;
+        }
+        if(wFixed && (1ULL << (paramConstraints.windowLog - 1)) > srcSize) {
+            memoTable[i] = 255;
+        }
+        /* nil out parameter sets equivalent to others. */
+        if(cwFixed/* at most least 1 param fixed. */) {
+            if(paramConstraints.strategy == ZSTD_btlazy2 || paramConstraints.strategy == ZSTD_btopt || paramConstraints.strategy == ZSTD_btultra) {
+                if(paramConstraints.chainLog > paramConstraints.windowLog + 1) {
+                    if(memoTable[i] != 255) { j++; }
+                    memoTable[i] = 255;
+                }
+            } else {
+                if(paramConstraints.chainLog > paramConstraints.windowLog) {
+                    if(memoTable[i] != 255) { j++; }
+                    memoTable[i] = 255;
+                }
+            }
+        }
+
+        if(scFixed) {
+            if(paramConstraints.searchLog > paramConstraints.chainLog) {
+                if(memoTable[i] != 255) { j++; }
+                memoTable[i] = 255;
+            }
+        }
+
+        if(whFixed) {
+            if(paramConstraints.hashLog > paramConstraints.windowLog + 1) {
+                if(memoTable[i] != 255) { j++; }
+                memoTable[i] = 255;
+            }
+        }
+    }
+    DEBUGOUTPUT("%d / %d Invalid\n", j, (int)i);
+    if((int)i == j) {
+        DEBUGOUTPUT("!!!Strategy %d totally infeasible\n", (int)paramConstraints.strategy)
+    }
+}
+
+/* frees all allocated memotables */
+static void freeMemoTableArray(U8** mtAll) {
+    int i;
+    if(mtAll == NULL) { return; }
+    for(i = 1; i <= (int)ZSTD_btultra; i++) {
+        free(mtAll[i]);
+    }
+    free(mtAll);
+}
+
+/* inits memotables for all (including mallocs), all strategies */
+/* takes unsanitized varyParams */
+static U8** createMemoTableArray(ZSTD_compressionParameters paramConstraints, constraint_t target, const varInds_t* varyParams, const int varyLen, const size_t srcSize) {
+    varInds_t varNew[NUM_PARAMS];
+    U8** mtAll = calloc(sizeof(U8*),(ZSTD_btultra + 1));
+    int i;
+    if(mtAll == NULL) {
+        return NULL;
+    }
+
+    for(i = 1; i <= (int)ZSTD_btultra; i++) {
+        const int varLenNew = sanitizeVarArray(varNew, varyLen, varyParams, i);
+        mtAll[i] = malloc(sizeof(U8) * memoTableLen(varNew, varLenNew));
+        if(mtAll[i] == NULL) {
+            freeMemoTableArray(mtAll);
+            return NULL;
+        }
+        initMemoTable(mtAll[i], paramConstraints, target, varNew, varLenNew, srcSize);
+    }
+    
+    return mtAll;
+}
 
 #define PARAMTABLELOG   25
 #define PARAMTABLESIZE (1<<PARAMTABLELOG)
 #define PARAMTABLEMASK (PARAMTABLESIZE-1)
 static BYTE g_alreadyTested[PARAMTABLESIZE] = {0};   /* init to zero */
 
+/*
 #define NB_TESTS_PLAYED(p) \
-    g_alreadyTested[(XXH64(sanitizeParams(p), sizeof(p), 0) >> 3) & PARAMTABLEMASK]
+    g_alreadyTested[(XXH64(((void*)&sanitizeParams(p), sizeof(p), 0) >> 3) & PARAMTABLEMASK] */
 
+static BYTE* NB_TESTS_PLAYED(ZSTD_compressionParameters p) {
+    ZSTD_compressionParameters p2 = sanitizeParams(p);
+    return &g_alreadyTested[(XXH64((void*)&p2, sizeof(p2), 0) >> 3) & PARAMTABLEMASK];
+}
 
 static void playAround(FILE* f, winnerInfo_t* winners,
                        ZSTD_compressionParameters params,
-                       const void* srcBuffer, size_t srcSize,
+                       const void* srcBuffer, size_t srcSize)
-                       ZSTD_CCtx* ctx, ZSTD_DCtx* dctx)
 {
     int nbVariations = 0;
     UTIL_time_t const clockStart = UTIL_getTime();
+    const U32 unconstrained[NUM_PARAMS] = { 0, 1, 2, 3, 4, 5 };
+
 
     while (UTIL_clockSpanMicro(clockStart) < g_maxVariationTime) {
         ZSTD_compressionParameters p = params;
+        BYTE* b;
 
         if (nbVariations++ > g_maxNbVariations) break;
-        paramVariation(&p);
+        paramVariation(&p, unconstrained, 7, 4);
 
         /* exclude faster if already played params */
-        if (FUZ_rand(&g_rand) & ((1 << NB_TESTS_PLAYED(p))-1))
+        if (FUZ_rand(&g_rand) & ((1 << *NB_TESTS_PLAYED(p))-1))
             continue;
 
         /* test */
-        NB_TESTS_PLAYED(p)++;
+        b = NB_TESTS_PLAYED(p);
-        if (!BMK_seed(winners, p, srcBuffer, srcSize, ctx, dctx)) continue;
+        (*b)++;
+        if (!BMK_seed(winners, p, srcBuffer, srcSize)) continue;
 
         /* improvement found => search more */
         BMK_printWinners(f, winners, srcSize);
-        playAround(f, winners, p, srcBuffer, srcSize, ctx, dctx);
+        playAround(f, winners, p, srcBuffer, srcSize);
     }
 
 }
 
-
+/* Completely random parameter selection */
 static ZSTD_compressionParameters randomParams(void)
 {
     ZSTD_compressionParameters p;
     U32 validated = 0;
     while (!validated) {
         /* totally random entry */
-        p.chainLog   = (FUZ_rand(&g_rand) % (ZSTD_CHAINLOG_MAX+1 - ZSTD_CHAINLOG_MIN)) + ZSTD_CHAINLOG_MIN;
+        p.chainLog   = (FUZ_rand(&g_rand) % (ZSTD_CHAINLOG_MAX+1 - ZSTD_CHAINLOG_MIN))         
-        p.hashLog    = (FUZ_rand(&g_rand) % (ZSTD_HASHLOG_MAX+1 - ZSTD_HASHLOG_MIN)) + ZSTD_HASHLOG_MIN;
+            + ZSTD_CHAINLOG_MIN;
-        p.searchLog  = (FUZ_rand(&g_rand) % (ZSTD_SEARCHLOG_MAX+1 - ZSTD_SEARCHLOG_MIN)) + ZSTD_SEARCHLOG_MIN;
+        p.hashLog    = (FUZ_rand(&g_rand) % (ZSTD_HASHLOG_MAX+1 - ZSTD_HASHLOG_MIN))           
-        p.windowLog  = (FUZ_rand(&g_rand) % (ZSTD_WINDOWLOG_MAX+1 - ZSTD_WINDOWLOG_MIN)) + ZSTD_WINDOWLOG_MIN;
+            + ZSTD_HASHLOG_MIN;
-        p.searchLength=(FUZ_rand(&g_rand) % (ZSTD_SEARCHLENGTH_MAX+1 - ZSTD_SEARCHLENGTH_MIN)) + ZSTD_SEARCHLENGTH_MIN;
+        p.searchLog  = (FUZ_rand(&g_rand) % (ZSTD_SEARCHLOG_MAX+1 - ZSTD_SEARCHLOG_MIN))       
+            + ZSTD_SEARCHLOG_MIN;
+        p.windowLog  = (FUZ_rand(&g_rand) % (ZSTD_WINDOWLOG_MAX+1 - ZSTD_WINDOWLOG_MIN))       
+            + ZSTD_WINDOWLOG_MIN;
+        p.searchLength=(FUZ_rand(&g_rand) % (ZSTD_SEARCHLENGTH_MAX+1 - ZSTD_SEARCHLENGTH_MIN)) 
+            + ZSTD_SEARCHLENGTH_MIN;
         p.targetLength=(FUZ_rand(&g_rand) % (512));
+
         p.strategy   = (ZSTD_strategy) (FUZ_rand(&g_rand) % (ZSTD_btultra +1));
+        
         validated = !ZSTD_isError(ZSTD_checkCParams(p));
     }
     return p;
 }
 
+/* Sets pc to random unmeasured set of parameters */
+static void randomConstrainedParams(ZSTD_compressionParameters* pc, varInds_t* varArray, int varLen, U8* memoTable)
+{
+    size_t tries = memoTableLen(varArray, varLen); 
+    const size_t maxSize = memoTableLen(varArray, varLen);
+    size_t ind;
+    do {
+        ind = (FUZ_rand(&g_rand)) % maxSize;
+        tries--;
+    } while(memoTable[ind] > 0 && tries > 0); 
+
+    memoTableIndInv(pc, varArray, varLen, (unsigned)ind);
+}
+
 static void BMK_selectRandomStart(
                        FILE* f, winnerInfo_t* winners,
-                       const void* srcBuffer, size_t srcSize,
+                       const void* srcBuffer, size_t srcSize)
-                       ZSTD_CCtx* ctx, ZSTD_DCtx* dctx)
 {
     U32 const id = FUZ_rand(&g_rand) % (NB_LEVELS_TRACKED+1);
     if ((id==0) || (winners[id].params.windowLog==0)) {
         /* use some random entry */
         ZSTD_compressionParameters const p = ZSTD_adjustCParams(randomParams(), srcSize, 0);
-        playAround(f, winners, p, srcBuffer, srcSize, ctx, dctx);
+        playAround(f, winners, p, srcBuffer, srcSize);
     } else {
-        playAround(f, winners, winners[id].params, srcBuffer, srcSize, ctx, dctx);
+        playAround(f, winners, winners[id].params, srcBuffer, srcSize);
     }
 }
 
 
-static void BMK_benchOnce(ZSTD_CCtx* cctx, ZSTD_DCtx* dctx, const void* srcBuffer, size_t srcSize)
+static void BMK_benchOnce(const void* srcBuffer, size_t srcSize)
 {
     BMK_result_t testResult;
     g_params = ZSTD_adjustCParams(g_params, srcSize, 0);
-    BMK_benchParam(&testResult, srcBuffer, srcSize, cctx, dctx, g_params);
+    BMK_benchParam1(&testResult, srcBuffer, srcSize, g_params);
     DISPLAY("Compression Ratio: %.3f  Compress Speed: %.1f MB/s Decompress Speed: %.1f MB/s\n", (double)srcSize / testResult.cSize, 
-        testResult.cSpeed / 1000000, testResult.dSpeed / 1000000);
+        (double)testResult.cSpeed / (1 MB), (double)testResult.dSpeed / (1 MB));
     return;
 }
 
-static void BMK_benchFullTable(ZSTD_CCtx* cctx, ZSTD_DCtx* dctx, const void* srcBuffer, size_t srcSize)
+static void BMK_benchFullTable(const void* srcBuffer, size_t srcSize)
 {
     ZSTD_compressionParameters params;
     winnerInfo_t winners[NB_LEVELS_TRACKED+1];
@@ -499,12 +1212,12 @@ static void BMK_benchFullTable(ZSTD_CCtx* cctx, ZSTD_DCtx* dctx, const void* src
     if (f==NULL) { DISPLAY("error opening %s \n", rfName); exit(1); }
 
     if (g_target) {
-        BMK_init_level_constraints(g_target*1000000);
+        BMK_init_level_constraints(g_target*(1 MB));
     } else {
         /* baseline config for level 1 */
         ZSTD_compressionParameters const l1params = ZSTD_getCParams(1, blockSize, 0);
         BMK_result_t testResult;
-        BMK_benchParam(&testResult, srcBuffer, srcSize, cctx, dctx, l1params);
+        BMK_benchParam1(&testResult, srcBuffer, srcSize, l1params);
         BMK_init_level_constraints((int)((testResult.cSpeed * 31) / 32));
     }
 
@@ -513,14 +1226,14 @@ static void BMK_benchFullTable(ZSTD_CCtx* cctx, ZSTD_DCtx* dctx, const void* src
         int i;
         for (i=0; i<=maxSeeds; i++) {
             params = ZSTD_getCParams(i, blockSize, 0);
-            BMK_seed(winners, params, srcBuffer, srcSize, cctx, dctx);
+            BMK_seed(winners, params, srcBuffer, srcSize);
     }   }
     BMK_printWinners(f, winners, srcSize);
 
     /* start tests */
     {   const time_t grillStart = time(NULL);
         do {
-            BMK_selectRandomStart(f, winners, srcBuffer, srcSize, cctx, dctx);
+            BMK_selectRandomStart(f, winners, srcBuffer, srcSize);
         } while (BMK_timeSpan(grillStart) < g_grillDuration_s);
     }
 
@@ -532,28 +1245,19 @@ static void BMK_benchFullTable(ZSTD_CCtx* cctx, ZSTD_DCtx* dctx, const void* src
     fclose(f);
 }
 
-static void BMK_benchMem_usingCCtx(ZSTD_CCtx* const cctx, ZSTD_DCtx* const dctx, const void* srcBuffer, size_t srcSize)
+static void BMK_benchMemInit(const void* srcBuffer, size_t srcSize)
 {
     if (g_singleRun)
-        return BMK_benchOnce(cctx, dctx, srcBuffer, srcSize);
+        return BMK_benchOnce(srcBuffer, srcSize);
     else
-        return BMK_benchFullTable(cctx, dctx, srcBuffer, srcSize);
+        return BMK_benchFullTable(srcBuffer, srcSize);
-}
-
-static void BMK_benchMemCCtxInit(const void* srcBuffer, size_t srcSize)
-{
-    ZSTD_CCtx* const cctx = ZSTD_createCCtx();
-    ZSTD_DCtx* const dctx = ZSTD_createDCtx();
-    if (cctx==NULL || dctx==NULL) { DISPLAY("Context Creation failed \n"); exit(1); }
-    BMK_benchMem_usingCCtx(cctx, dctx, srcBuffer, srcSize);
-    ZSTD_freeCCtx(cctx);
 }
 
 
 static int benchSample(void)
 {
     const char* const name = "Sample 10MB";
-    size_t const benchedSize = 10000000;
+    size_t const benchedSize = (10 MB);
 
     void* origBuff = malloc(benchedSize);
     if (!origBuff) { perror("not enough memory"); return 12; }
@@ -564,7 +1268,7 @@ static int benchSample(void)
     /* bench */
     DISPLAY("\r%79s\r", "");
     DISPLAY("using %s %i%%: \n", name, (int)(g_compressibility*100));
-    BMK_benchMemCCtxInit(origBuff, benchedSize);
+    BMK_benchMemInit(origBuff, benchedSize);
 
     free(origBuff);
     return 0;
@@ -622,7 +1326,7 @@ int benchFiles(const char** fileNamesTable, int nbFiles)
         /* bench */
         DISPLAY("\r%79s\r", "");
         DISPLAY("using %s : \n", inFileName);
-        BMK_benchMemCCtxInit(origBuff, benchedSize);
+        BMK_benchMemInit(origBuff, benchedSize);
 
         /* clean */
         free(origBuff);
@@ -632,131 +1336,715 @@ int benchFiles(const char** fileNamesTable, int nbFiles)
 }
 
 
-static void BMK_translateAdvancedParams(ZSTD_compressionParameters params)
+
-{
+#define WORSE_RESULT 0
-    DISPLAY("--zstd=windowLog=%u,chainLog=%u,hashLog=%u,searchLog=%u,searchLength=%u,targetLength=%u,strategy=%u \n",
+#define BETTER_RESULT 1
-             params.windowLog, params.chainLog, params.hashLog, params.searchLog, params.searchLength, params.targetLength, (U32)(params.strategy));
+#define ERROR_RESULT 2
+
+/* Benchmarking which stops when we are sufficiently sure the solution is infeasible / worse than the winner */
+#define VARIANCE 1.1 
+static int allBench(BMK_result_t* resultPtr,
+                const buffers_t buf, const contexts_t ctx,
+                const ZSTD_compressionParameters cParams,
+                const constraint_t target,
+                BMK_result_t* winnerResult, int feas) {
+    BMK_return_t benchres;
+    BMK_result_t resultMax;
+    U64 loopDurationC = 0, loopDurationD = 0;
+    double uncertaintyConstantC, uncertaintyConstantD;
+    double winnerRS;
+
+    /* initial benchmarking, gives exact ratio and memory, warms up future runs */
+    benchres = BMK_benchMemInvertible(buf, ctx, BASE_CLEVEL, &cParams, BMK_both, BMK_iterMode, 1);
+
+    winnerRS = resultScore(*winnerResult, buf.srcSize, target);
+    DEBUGOUTPUT("WinnerScore: %f\n ", winnerRS);
+
+    if(benchres.error) {
+        DEBUGOUTPUT("Benchmarking failed\n");
+        return ERROR_RESULT;
+    } 
+    *resultPtr = benchres.result;
+
+    /* calculate uncertainty in compression / decompression runs */
+    if(benchres.result.cSpeed) {
+        loopDurationC = ((buf.srcSize * TIMELOOP_NANOSEC) / benchres.result.cSpeed); 
+        uncertaintyConstantC = ((loopDurationC + (double)(2 * g_clockGranularity))/loopDurationC) * VARIANCE; 
+    } else {
+        loopDurationC = 0;
+        uncertaintyConstantC = 3;
+    }
+
+    if(benchres.result.dSpeed) {
+        loopDurationD = ((buf.srcSize * TIMELOOP_NANOSEC) / benchres.result.dSpeed); 
+        uncertaintyConstantD = ((loopDurationD + (double)(2 * g_clockGranularity))/loopDurationD) * VARIANCE;  
+    } else {
+        loopDurationD = 0;
+        uncertaintyConstantD = 3;
+    }
+
+    /* anything with worse ratio in feas is definitely worse, discard */
+    if(feas && benchres.result.cSize < winnerResult->cSize) {
+        return WORSE_RESULT;
+    }
+
+    /* second run, if first run is too short, gives approximate cSpeed + dSpeed */
+    if(loopDurationC < TIMELOOP_NANOSEC / 10) {
+        BMK_return_t benchres2 = BMK_benchMemInvertible(buf, ctx, BASE_CLEVEL, &cParams, BMK_compressOnly, BMK_iterMode, 1);
+        if(benchres2.error) {
+            return ERROR_RESULT;
+        }
+        benchres = benchres2;
+    }
+    if(loopDurationD < TIMELOOP_NANOSEC / 10) {
+        BMK_return_t benchres2 = BMK_benchMemInvertible(buf, ctx, BASE_CLEVEL, &cParams, BMK_decodeOnly, BMK_iterMode, 1);
+        if(benchres2.error) {
+            return ERROR_RESULT;
+        }
+        benchres.result.dSpeed = benchres2.result.dSpeed;
+    }
+
+    *resultPtr = benchres.result;
+
+    /* optimistic assumption of benchres.result */
+    resultMax = benchres.result;
+    resultMax.cSpeed *= uncertaintyConstantC;
+    resultMax.dSpeed *= uncertaintyConstantD;
+
+    /* disregard infeasible results in feas mode */
+    /* disregard if resultMax < winner in infeas mode */
+    if((feas && !feasible(resultMax, target)) || 
+      (!feas && (winnerRS > resultScore(resultMax, buf.srcSize, target)))) {
+        return WORSE_RESULT;
+    }
+
+    /* Final full run if estimates are unclear */
+    if(loopDurationC < TIMELOOP_NANOSEC) {
+        BMK_return_t benchres2 = BMK_benchMemInvertible(buf, ctx, BASE_CLEVEL, &cParams, BMK_compressOnly, BMK_timeMode, 1);
+        if(benchres2.error) {
+            return ERROR_RESULT;
+        }
+        benchres.result.cSpeed = benchres2.result.cSpeed;
+    } 
+
+    if(loopDurationD < TIMELOOP_NANOSEC) {
+        BMK_return_t benchres2 = BMK_benchMemInvertible(buf, ctx, BASE_CLEVEL, &cParams, BMK_decodeOnly, BMK_timeMode, 1);
+        if(benchres2.error) {
+            return ERROR_RESULT;
+        }
+        benchres.result.dSpeed = benchres2.result.dSpeed;
+    }
+
+    *resultPtr = benchres.result;
+
+    /* compare by resultScore when in infeas */
+    /* compare by compareResultLT when in feas */
+    if((!feas && (resultScore(benchres.result, buf.srcSize, target) > resultScore(*winnerResult, buf.srcSize, target))) || 
+       (feas && (compareResultLT(*winnerResult, benchres.result, target, buf.srcSize))) )  { 
+        return BETTER_RESULT; 
+    } else { 
+        return WORSE_RESULT; 
+    }
+
 }
 
-/* optimizeForSize():
+#define INFEASIBLE_THRESHOLD 200
- * targetSpeed : expressed in MB/s */
+
-int optimizeForSize(const char* inFileName, U32 targetSpeed)
+/* Memoized benchmarking, won't benchmark anything which has already been benchmarked before. */
+static int benchMemo(BMK_result_t* resultPtr,
+                const buffers_t buf, const contexts_t ctx, 
+                const ZSTD_compressionParameters cParams,
+                const constraint_t target,
+                BMK_result_t* winnerResult, U8* const memoTable,
+                const varInds_t* varyParams, const int varyLen, const int feas) {
+    static int bmcount = 0;
+    size_t memind = memoTableInd(&cParams, varyParams, varyLen);
+    int res;
+
+    if(memoTable[memind] >= INFEASIBLE_THRESHOLD) { return WORSE_RESULT; } 
+
+    res = allBench(resultPtr, buf, ctx, cParams, target, winnerResult, feas);
+
+    if(DEBUG && !(bmcount % 250)) {
+        DISPLAY("Count: %d\n", bmcount);
+        bmcount++;
+    }
+    BMK_printWinnerOpt(stdout, CUSTOM_LEVEL, *resultPtr, cParams, target, buf.srcSize);
+
+    if(res == BETTER_RESULT || feas) {
+        memoTable[memind] = 255; 
+    }
+    return res;
+}
+
+/* One iteration of hill climbing. Specifically, it first tries all 
+ * valid parameter configurations w/ manhattan distance 1 and picks the best one
+ * failing that, it progressively tries candidates further and further away (up to #dim + 2)
+ * if it finds a candidate exceeding winnerInfo, it will repeat. Otherwise, it will stop the 
+ * current stage of hill climbing. 
+ * Each iteration of hill climbing proceeds in 2 'phases'. Phase 1 climbs according to 
+ * the resultScore function, which is effectively a linear increase in reward until it reaches
+ * the constraint-satisfying value, it which point any excess results in only logarithmic reward.
+ * This aims to find some constraint-satisfying point.
+ * Phase 2 optimizes in accordance with what the original function sets out to maximize, with
+ * all feasible solutions valued over all infeasible solutions.
+ */
+static winnerInfo_t climbOnce(const constraint_t target, 
+                const varInds_t* varArray, const int varLen, 
+                U8* const memoTable,
+                const buffers_t buf, const contexts_t ctx,
+                const ZSTD_compressionParameters init) {
+    /* 
+     * cparam - currently considered 'center'
+     * candidate - params to benchmark/results
+     * winner - best option found so far.
+     */
+    ZSTD_compressionParameters cparam = init;
+    winnerInfo_t candidateInfo, winnerInfo;
+    int better = 1;
+    int feas = 0;
+
+    winnerInfo = initWinnerInfo(init);
+    candidateInfo = winnerInfo;
+
+    {
+        winnerInfo_t bestFeasible1 = initWinnerInfo(cparam);
+        DISPLAY("Climb Part 1\n");
+        while(better) {
+
+            int i, dist, offset;
+            better = 0;
+            DEBUGOUTPUT("Start\n");
+            cparam = winnerInfo.params;
+            BMK_printWinnerOpt(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, target, buf.srcSize);
+            candidateInfo.params = cparam;
+             /* all dist-1 candidates */
+            for(i = 0; i < varLen; i++) {
+                for(offset = -1; offset <= 1; offset += 2) {
+                    candidateInfo.params = cparam;
+                    paramVaryOnce(varArray[i], offset, &candidateInfo.params); 
+                    candidateInfo.params = sanitizeParams(candidateInfo.params);
+                    if(!ZSTD_isError(ZSTD_checkCParams(candidateInfo.params))) {
+                        int res = benchMemo(&candidateInfo.result,
+                            buf, ctx,
+                            candidateInfo.params, target, &winnerInfo.result, memoTable,
+                            varArray, varLen, feas);
+                        if(res == BETTER_RESULT) { /* synonymous with better when called w/ infeasibleBM */
+                            winnerInfo = candidateInfo;
+                            BMK_printWinnerOpt(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, target, buf.srcSize);
+                            better = 1;
+                            if(compareResultLT(bestFeasible1.result, winnerInfo.result, target, buf.srcSize)) {
+                                bestFeasible1 = winnerInfo;
+                            }
+                        }
+                    }
+                }
+            }
+
+            if(better) {
+                continue;
+            }
+
+            for(dist = 2; dist < varLen + 2; dist++) { /* varLen is # dimensions */
+                for(i = 0; i < 2 * varLen + 2; i++) {
+                    int res;
+                    candidateInfo.params = cparam;
+                    /* param error checking already done here */
+                    paramVariation(&candidateInfo.params, varArray, varLen, dist);
+                    res = benchMemo(&candidateInfo.result,
+                        buf, ctx, 
+                        candidateInfo.params, target, &winnerInfo.result, memoTable,
+                        varArray, varLen, feas);
+                    if(res == BETTER_RESULT) { /* synonymous with better in this case*/
+                        winnerInfo = candidateInfo;
+                        BMK_printWinnerOpt(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, target, buf.srcSize);
+                        better = 1;
+                        if(compareResultLT(bestFeasible1.result, winnerInfo.result, target, buf.srcSize)) {
+                            bestFeasible1 = winnerInfo;
+                        }
+                    }
+
+                }
+                if(better) {
+                    break;
+                }
+            }
+
+            if(!better) { /* infeas -> feas -> stop */ 
+                if(feas) { return winnerInfo; } 
+
+                feas = 1;
+                better = 1;
+                winnerInfo = bestFeasible1; /* note with change, bestFeasible may not necessarily be feasible, but if one has been benchmarked, it will be. */
+                DISPLAY("Climb Part 2\n");
+            }
+        }
+        winnerInfo = bestFeasible1;
+    }
+
+    return winnerInfo;
+}
+
+/* Optimizes for a fixed strategy */
+
+/* flexible parameters: iterations of (failed?) climbing (or if we do non-random, maybe this is when everything is close to visitied)
+   weight more on visit for bad results, less on good results/more on later results / ones with more failures.
+   allocate memoTable here. 
+   only real use for paramTarget is to get the fixed values, right? 
+   maybe allow giving it a first init? 
+ */
+static winnerInfo_t optimizeFixedStrategy(
+    const buffers_t buf, const contexts_t ctx, 
+    const constraint_t target, ZSTD_compressionParameters paramTarget,
+    const ZSTD_strategy strat, 
+    const varInds_t* varArray, const int varLen,
+    U8* const memoTable, const int tries) {
+    int i = 0;
+    varInds_t varNew[NUM_PARAMS];
+    int varLenNew = sanitizeVarArray(varNew, varLen, varArray, strat);
+    ZSTD_compressionParameters init;
+    winnerInfo_t winnerInfo, candidateInfo; 
+    winnerInfo = initWinnerInfo(emptyParams());
+    /* so climb is given the right fixed strategy */
+    paramTarget.strategy = strat;
+    /* to pass ZSTD_checkCParams */
+
+    cParamZeroMin(&paramTarget);
+
+    init = paramTarget;
+
+    while(i < tries) {
+        DEBUGOUTPUT("Restart\n"); 
+        randomConstrainedParams(&init, varNew, varLenNew, memoTable);
+        candidateInfo = climbOnce(target, varNew, varLenNew, memoTable, buf, ctx, init);
+        if(compareResultLT(winnerInfo.result, candidateInfo.result, target, buf.srcSize)) {
+            winnerInfo = candidateInfo;
+            BMK_printWinnerOpt(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, target, buf.srcSize);
+            i = 0;
+        }
+        i++;
+    }
+    return winnerInfo;
+}
+
+static void freeBuffers(const buffers_t b) {
+    if(b.srcPtrs != NULL) {
+        free(b.srcBuffer);
+    }
+    free(b.srcPtrs);
+    free(b.srcSizes);
+
+    if(b.dstPtrs != NULL) {
+        free(b.dstPtrs[0]);
+    }
+    free(b.dstPtrs);
+    free(b.dstCapacities);
+    free(b.dstSizes);
+
+    if(b.resPtrs != NULL) {
+        free(b.resPtrs[0]);
+    }
+    free(b.resPtrs);
+}
+
+/* allocates buffer's arguments. returns 0 = success / 1 = failuere */
+static int createBuffers(buffers_t* const buff, const char* const * const fileNamesTable, 
+                          const size_t nbFiles)
 {
-    FILE* const inFile = fopen( inFileName, "rb" );
+    size_t pos = 0;
-    U64 const inFileSize = UTIL_getFileSize(inFileName);
+    size_t n;
-    size_t benchedSize = BMK_findMaxMem(inFileSize*3) / 3;
+    U64 const totalSizeToLoad = UTIL_getTotalFileSize(fileNamesTable, (U32)nbFiles);
-    void* origBuff;
+    size_t benchedSize = MIN(BMK_findMaxMem(totalSizeToLoad * 3) / 3, totalSizeToLoad);
+    const size_t blockSize = g_blockSize ? g_blockSize : totalSizeToLoad; //(largest fileSize or total fileSize)
+    U32 const maxNbBlocks = (U32) ((totalSizeToLoad + (blockSize-1)) / blockSize) + (U32)nbFiles;
+    U32 blockNb = 0;
+
+    memset(buff, 0, sizeof(buffers_t));
+
+    buff->srcPtrs = (const void**)calloc(maxNbBlocks, sizeof(void*));
+    buff->srcSizes = (size_t*)malloc(maxNbBlocks * sizeof(size_t));
+
+    buff->dstPtrs = (void**)calloc(maxNbBlocks, sizeof(void*));
+    buff->dstCapacities = (size_t*)malloc(maxNbBlocks * sizeof(size_t));
+    buff->dstSizes = (size_t*)malloc(maxNbBlocks * sizeof(size_t));
+
+    buff->resPtrs = (void**)calloc(maxNbBlocks, sizeof(void*));
+    buff->resSizes = (size_t*)malloc(maxNbBlocks * sizeof(size_t)); 
+
+    if(!buff->srcPtrs || !buff->srcSizes || !buff->dstPtrs || !buff->dstCapacities || !buff->dstSizes || !buff->resPtrs || !buff->resSizes) {
+        DISPLAY("alloc error\n");
+        freeBuffers(*buff);
+        return 1;
+    }
+
+    buff->srcBuffer = malloc(benchedSize);
+    buff->srcPtrs[0] = (const void*)buff->srcBuffer;
+    buff->dstPtrs[0] = malloc(ZSTD_compressBound(benchedSize) + (maxNbBlocks * 1024));
+    buff->resPtrs[0] = malloc(benchedSize);
+
+    if(!buff->srcPtrs[0] || !buff->dstPtrs[0] || !buff->resPtrs[0]) {
+        DISPLAY("alloc error\n");
+        freeBuffers(*buff);
+        return 1;
+    }
+
+    for(n = 0; n < nbFiles; n++) {
+        FILE* f;
+        U64 fileSize = UTIL_getFileSize(fileNamesTable[n]);
+        if (UTIL_isDirectory(fileNamesTable[n])) {
+            DISPLAY("Ignoring %s directory...       \n", fileNamesTable[n]);
+            continue;
+        }
+        if (fileSize == UTIL_FILESIZE_UNKNOWN) {
+            DISPLAY("Cannot evaluate size of %s, ignoring ... \n", fileNamesTable[n]);
+            continue;
+        }
+        f = fopen(fileNamesTable[n], "rb");
+        if (f==NULL) {
+            DISPLAY("impossible to open file %s\n", fileNamesTable[n]);
+            freeBuffers(*buff);
+            fclose(f);
+            return 10;
+        }
+
+        DISPLAY("Loading %s...       \r", fileNamesTable[n]);
+
+        if (fileSize + pos > benchedSize) fileSize = benchedSize - pos, n = nbFiles;   /* buffer too small - stop after this file */
+        {
+            char* buffer = (char*)(buff->srcBuffer); 
+            size_t const readSize = fread(((buffer)+pos), 1, (size_t)fileSize, f);
+            size_t blocked = 0;
+            while(blocked < readSize) {
+                buff->srcPtrs[blockNb] = (const void*)((buffer) + (pos + blocked));
+                buff->srcSizes[blockNb] = blockSize;
+                blocked += blockSize;
+                blockNb++;
+            }
+            if(readSize > 0) { buff->srcSizes[blockNb - 1] = ((readSize - 1) % blockSize) + 1; }
+
+            if (readSize != (size_t)fileSize) {
+                DISPLAY("could not read %s", fileNamesTable[n]);
+                freeBuffers(*buff);
+                fclose(f);
+                return 1;
+            }
+
+            pos += readSize;
+
+        }
+        fclose(f);
+    }
+
+    buff->dstCapacities[0] = ZSTD_compressBound(buff->srcSizes[0]);
+    buff->dstSizes[0] = buff->dstCapacities[0];
+    buff->resSizes[0] = buff->srcSizes[0];
+
+    for(n = 1; n < blockNb; n++) {
+        buff->dstPtrs[n] = ((char*)buff->dstPtrs[n-1]) + buff->dstCapacities[n-1];
+        buff->resPtrs[n] = ((char*)buff->resPtrs[n-1]) + buff->resSizes[n-1];
+        buff->dstCapacities[n] = ZSTD_compressBound(buff->srcSizes[n]);
+        buff->dstSizes[n] = buff->dstCapacities[n];
+        buff->resSizes[n] = buff->srcSizes[n];
+    }
+    buff->srcSize = pos;
+    buff->nbBlocks = blockNb;
+
+    if (pos == 0) { DISPLAY("\nno data to bench\n"); return 1; }
+
+    return 0;
+}
+
+static void freeContexts(const contexts_t ctx) {
+    free(ctx.dictBuffer);
+    ZSTD_freeCCtx(ctx.cctx);
+    ZSTD_freeDCtx(ctx.dctx);
+}
+
+/* Creates struct holding contexts and dictionary buffers. returns 0 on success, 1 on failure. */
+static int createContexts(contexts_t* const ctx, const char* dictFileName) {
+    FILE* f;
+    size_t readSize;
+    U64 dictSize;
+    ctx->cctx = ZSTD_createCCtx();
+    ctx->dctx = ZSTD_createDCtx();
+    ctx->dictSize = 0;
+    ctx->dictBuffer = NULL;
+
+    if(!ctx->cctx || !ctx->dctx) {
+        DISPLAY("context allocation error\n");
+        freeContexts(*ctx);
+        return 1;
+    }
+
+    if(dictFileName == NULL) {
+        return 0;
+    }
+
+    dictSize = UTIL_getFileSize(dictFileName);
+
+    if(dictSize == UTIL_FILESIZE_UNKNOWN) {
+        DISPLAY("Unable to get dictionary size\n");
+        freeContexts(*ctx);
+        return 1;
+    } else {
+        ctx->dictSize = (size_t)dictSize;
+    }
+
+    ctx->dictBuffer = malloc(ctx->dictSize);
+
+    f = fopen(dictFileName, "rb");
+    
+    if(!f) {
+        DISPLAY("unable to open file\n");
+        fclose(f);
+        freeContexts(*ctx);
+        return 1;
+    }
+    
+    if(ctx->dictSize > 64 MB || !(ctx->dictBuffer)) {
+        DISPLAY("dictionary too large\n");
+        fclose(f);
+        freeContexts(*ctx);
+        return 1;
+    }
+    readSize = fread(ctx->dictBuffer, 1, ctx->dictSize, f);
+    if(readSize != ctx->dictSize) {
+        DISPLAY("unable to read file\n");
+        fclose(f);
+        freeContexts(*ctx);
+        return 1;
+    }
+    return 0;
+}
+
+/* goes best, best-1, best+1, best-2, ... */
+/* return 0 if nothing remaining */
+static int nextStrategy(const int currentStrategy, const int bestStrategy) {
+    if(bestStrategy <= currentStrategy) {
+        int candidate = 2 * bestStrategy - currentStrategy - 1;
+        if(candidate < 1) {
+            candidate = currentStrategy + 1;
+            if(candidate > (int)ZSTD_btultra) {
+                return 0;
+            } else {
+                return candidate;
+            }
+        } else {
+            return candidate;
+        }
+    } else { /* bestStrategy >= currentStrategy */
+        int candidate = 2 * bestStrategy - currentStrategy;
+        if(candidate > (int)ZSTD_btultra) {
+            candidate = currentStrategy - 1;
+            if(candidate < 1) {
+                return 0;
+            } else {
+                return candidate;
+            }
+        } else {
+            return candidate;
+        }
+    }
+}
+
+static ZSTD_compressionParameters maskParams(ZSTD_compressionParameters base, ZSTD_compressionParameters mask) {
+    base.windowLog = mask.windowLog ? mask.windowLog : base.windowLog;
+    base.chainLog = mask.chainLog ? mask.chainLog : base.chainLog;
+    base.hashLog = mask.hashLog ? mask.hashLog : base.hashLog;
+    base.searchLog = mask.searchLog ? mask.searchLog : base.searchLog;
+    base.searchLength = mask.searchLength ? mask.searchLength : base.searchLength;
+    base.targetLength = mask.targetLength ? mask.targetLength : base.targetLength;
+    base.strategy = mask.strategy ? mask.strategy : base.strategy;
+    return base;
+}
+
+#define MAX_TRIES 8
+/* main fn called when using --optimize */
+/* Does strategy selection by benchmarking default compression levels
+ * then optimizes by strategy, starting with the best one and moving 
+ * progressively moving further away by number
+ * args: 
+ * fileNamesTable - list of files to benchmark
+ * nbFiles - length of fileNamesTable
+ * dictFileName - name of dictionary file if one, else NULL
+ * target - performance constraints (cSpeed, dSpeed, cMem)
+ * paramTarget - parameter constraints (i.e. restriction search space to where strategy = ZSTD_fast)
+ * cLevel - compression level to exceed (all solutions must be > lvl in cSpeed + ratio)
+ */
+static int optimizeForSize(const char* const * const fileNamesTable, const size_t nbFiles, const char* dictFileName, constraint_t target, ZSTD_compressionParameters paramTarget, int cLevel)
+{
+    varInds_t varArray [NUM_PARAMS];
+    int ret = 0;
+    const int varLen = variableParams(paramTarget, varArray);
+    winnerInfo_t winner = initWinnerInfo(emptyParams());
+    U8** allMT = NULL;
+    size_t k;
+    size_t maxBlockSize = 0;
+    contexts_t ctx;
+    buffers_t buf;
+
     /* Init */
-    if (inFile==NULL) { DISPLAY( "Pb opening %s\n", inFileName); return 11; }
+    if(!cParamValid(paramTarget)) {
-    if (inFileSize == UTIL_FILESIZE_UNKNOWN) {
+        return 1;
-        DISPLAY("Pb evaluatin size of %s \n", inFileName);
-        fclose(inFile);
-        return 11;
     }
 
-    /* Memory allocation & restrictions */
+    /* load dictionary*/
-    if ((U64)benchedSize > inFileSize) benchedSize = (size_t)inFileSize;
+    if(createBuffers(&buf, fileNamesTable, nbFiles)) {
-    if (benchedSize < inFileSize) {
+        DISPLAY("unable to load files\n");
-        DISPLAY("Not enough memory for '%s' \n", inFileName);
+        return 1;
-        fclose(inFile);
-        return 11;
     }
 
-    /* Alloc */
+    if(createContexts(&ctx, dictFileName)) {
-    origBuff = malloc(benchedSize);
+        DISPLAY("unable to load dictionary\n");
-    if(!origBuff) {
+        freeBuffers(buf);
-        DISPLAY("\nError: not enough memory!\n");
+        return 2;
-        fclose(inFile);
-        return 12;
     }
 
-    /* Fill input buffer */
+    if(nbFiles == 1) {
-    DISPLAY("Loading %s...       \r", inFileName);
+        DISPLAY("Loading %s...       \r", fileNamesTable[0]);
-    {   size_t const readSize = fread(origBuff, 1, benchedSize, inFile);
+    } else {
-        fclose(inFile);
+        DISPLAY("Loading %lu Files...       \r", (unsigned long)nbFiles); 
-        if(readSize != benchedSize) {
+    }
-            DISPLAY("\nError: problem reading file '%s' !!    \n", inFileName);
+
-            free(origBuff);
+
-            return 13;
+    for(k = 0; k < buf.nbBlocks; k++) {
-    }   }
+        maxBlockSize = MAX(buf.srcSizes[k], maxBlockSize);
+    }
+
+    /* if strategy is fixed, only init that part of memotable */
+    if(paramTarget.strategy) {
+        varInds_t varNew[NUM_PARAMS];
+        int varLenNew = sanitizeVarArray(varNew, varLen, varArray, paramTarget.strategy);
+        allMT = calloc(sizeof(U8), (ZSTD_btultra + 1));
+        if(allMT == NULL) {
+            ret = 57;
+            goto _cleanUp;
+        }
+
+        allMT[paramTarget.strategy] = malloc(sizeof(U8) * memoTableLen(varNew, varLenNew));
+        
+        if(allMT[paramTarget.strategy] == NULL) {
+            ret = 58;
+            goto _cleanUp;
+        }
+        
+        initMemoTable(allMT[paramTarget.strategy], paramTarget, target, varNew, varLenNew, maxBlockSize);
+    } else {
+         allMT = createMemoTableArray(paramTarget, target, varArray, varLen, maxBlockSize);
+    }
+   
+
+    if(!allMT) {
+        DISPLAY("MemoTable Init Error\n");
+        ret = 2;
+        goto _cleanUp;
+    }
+ 
+    if(cLevel) {
+        winner.params = ZSTD_getCParams(cLevel, maxBlockSize, ctx.dictSize);
+        if(BMK_benchParam(&winner.result, buf, ctx, winner.params)) {
+            ret = 3;
+            goto _cleanUp;
+        }
+
+        target.cSpeed = (U32)winner.result.cSpeed;
+        BMK_printWinnerOpt(stdout, cLevel, winner.result, winner.params, target, buf.srcSize);
+    }
 
     /* bench */
     DISPLAY("\r%79s\r", "");
-    DISPLAY("optimizing for %s - limit speed %u MB/s \n", inFileName, targetSpeed);
+    if(nbFiles == 1) {
-    targetSpeed *= 1000000;
+        DISPLAY("optimizing for %s", fileNamesTable[0]);
-    {   ZSTD_CCtx* const ctx = ZSTD_createCCtx();
+    } else {
-        ZSTD_DCtx* const dctx = ZSTD_createDCtx();
+        DISPLAY("optimizing for %lu Files", (unsigned long)nbFiles);
-        winnerInfo_t winner;
+    }
-        BMK_result_t candidate;
+    if(target.cSpeed != 0) { DISPLAY(" - limit compression speed %u MB/s", target.cSpeed >> 20); }
-        const size_t blockSize = g_blockSize ? g_blockSize : benchedSize;
+    if(target.dSpeed != 0) { DISPLAY(" - limit decompression speed %u MB/s", target.dSpeed >> 20); }
+    if(target.cMem != (U32)-1) { DISPLAY(" - limit memory %u MB", target.cMem >> 20); }
 
-        /* init */
+    DISPLAY("\n");
-        if (ctx==NULL) { DISPLAY("\n ZSTD_createCCtx error \n"); free(origBuff); return 14;}
+    findClockGranularity();
-        memset(&winner, 0, sizeof(winner));
+
-        winner.result.cSize = (size_t)(-1);
+    {   
+        varInds_t varNew[NUM_PARAMS];
 
         /* find best solution from default params */
-        {   const int maxSeeds = g_noSeed ? 1 : ZSTD_maxCLevel();
+        {
-            int i;
+            /* strategy selection */
-            for (i=1; i<=maxSeeds; i++) {
+            const int maxSeeds = g_noSeed ? 1 : ZSTD_maxCLevel();
-                ZSTD_compressionParameters const CParams = ZSTD_getCParams(i, blockSize, 0);
+            DEBUGOUTPUT("Strategy Selection\n");
-                BMK_benchParam(&candidate, origBuff, benchedSize, ctx, dctx, CParams);
+            if(paramTarget.strategy == 0) { /* no variable based constraints */  
-                if (candidate.cSpeed < (double)targetSpeed) {
+                BMK_result_t candidate;
-                    break;
+                int i;
+                for (i=1; i<=maxSeeds; i++) {
+                    int ec;
+                    ZSTD_compressionParameters CParams = ZSTD_getCParams(i, maxBlockSize, ctx.dictSize);
+                    CParams = maskParams(CParams, paramTarget);
+                    ec = BMK_benchParam(&candidate, buf, ctx, CParams);
+                    BMK_printWinnerOpt(stdout, i, candidate, CParams, target, buf.srcSize);
+
+                    if(!ec && compareResultLT(winner.result, candidate, relaxTarget(target), buf.srcSize)) {
+                        winner.result = candidate;
+                        winner.params = CParams;
+                    }
                 }
-                if ( (candidate.cSize < winner.result.cSize)
+            }
-                   | ((candidate.cSize == winner.result.cSize) & (candidate.cSpeed > winner.result.cSpeed)) )
-                {
-                    winner.params = CParams;
-                    winner.result = candidate;
-                    BMK_printWinner(stdout, i, winner.result, winner.params, benchedSize);
-            }   }
         }
 
-        BMK_printWinner(stdout, CUSTOM_LEVEL, winner.result, winner.params, benchedSize);
+        BMK_printWinnerOpt(stdout, CUSTOM_LEVEL, winner.result, winner.params, target, buf.srcSize);
-
         BMK_translateAdvancedParams(winner.params);
+        DEBUGOUTPUT("Real Opt\n");
+        /* start 'real' tests */
+        {   
+            int bestStrategy = (int)winner.params.strategy;
+            if(paramTarget.strategy == 0) {
+                int st = (int)winner.params.strategy;
+                int tries = MAX_TRIES;
 
-        /* start tests */
+                { 
-        {   time_t const grillStart = time(NULL);
+                    int varLenNew = sanitizeVarArray(varNew, varLen, varArray, st);
-            do {
+                    winnerInfo_t w1 = climbOnce(target, varNew, varLenNew, allMT[st], 
-                ZSTD_compressionParameters params = winner.params;
+                        buf, ctx, winner.params);
-                paramVariation(&params);
+                    if(compareResultLT(winner.result, w1.result, target, buf.srcSize)) {
-                if ((FUZ_rand(&g_rand) & 31) == 3) params = randomParams();  /* totally random config to improve search space */
+                        winner = w1;
-                params = ZSTD_adjustCParams(params, blockSize, 0);
+                    }
-
-                /* exclude faster if already played set of params */
-                if (FUZ_rand(&g_rand) & ((1 << NB_TESTS_PLAYED(params))-1)) continue;
-
-                /* test */
-                NB_TESTS_PLAYED(params)++;
-                BMK_benchParam(&candidate, origBuff, benchedSize, ctx, dctx, params);
-
-                /* improvement found => new winner */
-                if ( (candidate.cSpeed > targetSpeed)
-                   & ( (candidate.cSize < winner.result.cSize)
-                     | ((candidate.cSize == winner.result.cSize) & (candidate.cSpeed > winner.result.cSpeed)) )  )
-                {
-                    winner.params = params;
-                    winner.result = candidate;
-                    BMK_printWinner(stdout, CUSTOM_LEVEL, winner.result, winner.params, benchedSize);
-                    BMK_translateAdvancedParams(winner.params);
                 }
-            } while (BMK_timeSpan(grillStart) < g_grillDuration_s);
+
+                while(st && tries) {
+                    winnerInfo_t wc = optimizeFixedStrategy(buf, ctx, target, paramTarget, 
+                        st, varArray, varLen, allMT[st], tries);
+                    DEBUGOUTPUT("StratNum %d\n", st);
+                    if(compareResultLT(winner.result, wc.result, target, buf.srcSize)) {
+                        winner = wc;
+                    }
+
+                    st = nextStrategy(st, bestStrategy);
+                    tries--;
+                }
+            } else {
+                winner = optimizeFixedStrategy(buf, ctx, target, paramTarget, paramTarget.strategy, 
+                    varArray, varLen, allMT[paramTarget.strategy], 10);
+            }
+
+        }
+
+        /* no solution found */
+        if(winner.result.cSize == (size_t)-1) {
+            ret = 1;
+            DISPLAY("No feasible solution found\n");
+            goto _cleanUp;
         }
         /* end summary */
-
+        BMK_printWinnerOpt(stdout, CUSTOM_LEVEL, winner.result, winner.params, target, buf.srcSize);
-        BMK_printWinner(stdout, CUSTOM_LEVEL, winner.result, winner.params, benchedSize);
         BMK_translateAdvancedParams(winner.params);
         DISPLAY("grillParams size - optimizer completed \n");
 
-        /* clean up*/
-        ZSTD_freeCCtx(ctx);
-        ZSTD_freeDCtx(dctx);
     }
-
+_cleanUp: 
-    free(origBuff);
+    freeContexts(ctx);
-    return 0;
+    freeBuffers(buf);
+    freeMemoTableArray(allMT);
+    return ret;
 }
 
 static void errorOut(const char* msg)
@@ -806,15 +2094,16 @@ static int usage(const char* exename)
 static int usage_advanced(void)
 {
     DISPLAY( "\nAdvanced options :\n");
-    DISPLAY( " -T#    : set level 1 speed objective \n");
+    DISPLAY( " -T#          : set level 1 speed objective \n");
-    DISPLAY( " -B#    : cut input into blocks of size # (default : single block) \n");
+    DISPLAY( " -B#          : cut input into blocks of size # (default : single block) \n");
-    DISPLAY( " -i#    : iteration loops [1-9](default : %i) \n", NBLOOPS);
+    DISPLAY( " -i#          : iteration loops (default : %i) \n", NBLOOPS);
-    DISPLAY( " -O#    : find Optimized parameters for # MB/s compression speed (default : 0) \n");
+    DISPLAY( " --optimize=  : same as -O with more verbose syntax (see README.md)\n");
-    DISPLAY( " -S     : Single run \n");
+    DISPLAY( " -S           : Single run \n");
-    DISPLAY( " --zstd : Single run, parameter selection same as zstdcli \n");
+    DISPLAY( " --zstd       : Single run, parameter selection same as zstdcli \n");
-    DISPLAY( " -P#    : generated sample compressibility (default : %.1f%%) \n", COMPRESSIBILITY_DEFAULT * 100);
+    DISPLAY( " -P#          : generated sample compressibility (default : %.1f%%) \n", COMPRESSIBILITY_DEFAULT * 100);
-    DISPLAY( " -t#    : Caps runtime of operation in seconds (default : %u seconds (%.1f hours)) \n", (U32)g_grillDuration_s, g_grillDuration_s / 3600);
+    DISPLAY( " -t#          : Caps runtime of operation in seconds (default : %u seconds (%.1f hours)) \n", (U32)g_grillDuration_s, g_grillDuration_s / 3600);
-    DISPLAY( " -v     : Prints Benchmarking output\n");
+    DISPLAY( " -v           : Prints Benchmarking output\n");
+    DISPLAY( " -D           : Next argument dictionary file\n");
     return 0;
 }
 
@@ -825,40 +2114,73 @@ static int badusage(const char* exename)
     return 1;
 }
 
+#define PARSE_SUB_ARGS(stringLong, stringShort, variable) { if (longCommandWArg(&argument, stringLong) || longCommandWArg(&argument, stringShort)) { variable = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; } }
+#define PARSE_CPARAMS(variable)                                          \
+{                                                                        \
+    PARSE_SUB_ARGS("windowLog=",     "wlog=",  variable.windowLog);      \
+    PARSE_SUB_ARGS("chainLog=" ,     "clog=",  variable.chainLog);       \
+    PARSE_SUB_ARGS("hashLog=",       "hlog=",  variable.hashLog);        \
+    PARSE_SUB_ARGS("searchLog=" ,    "slog=",  variable.searchLog);      \
+    PARSE_SUB_ARGS("searchLength=",  "slen=",  variable.searchLength);   \
+    PARSE_SUB_ARGS("targetLength=" , "tlen=",  variable.targetLength);   \
+    PARSE_SUB_ARGS("strategy=",      "strat=", variable.strategy);       \
+}
+
 int main(int argc, const char** argv)
 {
     int i,
         filenamesStart=0,
         result;
     const char* exename=argv[0];
-    const char* input_filename=0;
+    const char* input_filename = NULL;
+    const char* dictFileName = NULL;
     U32 optimizer = 0;
     U32 main_pause = 0;
-    U32 targetSpeed = 0;
+    int optimizerCLevel = 0;
+
+
+    constraint_t target = { 0, 0, (U32)-1 }; 
+    ZSTD_compressionParameters paramTarget = { 0, 0, 0, 0, 0, 0, 0 };
 
     assert(argc>=1);   /* for exename */
 
+    g_time = UTIL_getTime();
+
     /* Welcome message */
     DISPLAY(WELCOME_MESSAGE);
 
     for(i=1; i<argc; i++) {
         const char* argument = argv[i];
+        DISPLAY("%d: ", i);
+        DISPLAY("%s\n", argument);
+
         assert(argument != NULL);
 
         if(!strcmp(argument,"--no-seed")) { g_noSeed = 1; continue; }
 
+        if (longCommandWArg(&argument, "--optimize=")) {
+            optimizer = 1;
+            for ( ; ;) {
+                PARSE_CPARAMS(paramTarget);
+                PARSE_SUB_ARGS("compressionSpeed=" ,  "cSpeed=", target.cSpeed);
+                PARSE_SUB_ARGS("decompressionSpeed=", "dSpeed=", target.dSpeed); 
+                PARSE_SUB_ARGS("compressionMemory=" , "cMem=", target.cMem);
+                PARSE_SUB_ARGS("level=", "lvl=", optimizerCLevel);
+                DISPLAY("invalid optimization parameter \n");
+                return 1;
+            }
+
+            if (argument[0] != 0) {
+                DISPLAY("invalid --optimize= format\n");
+                return 1; /* check the end of string */
+            }
+            continue;
+        } else if (longCommandWArg(&argument, "--zstd=")) {
         /* Decode command (note : aggregated commands are allowed) */
-        if (longCommandWArg(&argument, "--zstd=")) {
             g_singleRun = 1;
             g_params = ZSTD_getCParams(2, g_blockSize, 0);
             for ( ; ;) {
-                if (longCommandWArg(&argument, "windowLog=") || longCommandWArg(&argument, "wlog=")) { g_params.windowLog = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; }
+                PARSE_CPARAMS(g_params)
-                if (longCommandWArg(&argument, "chainLog=") || longCommandWArg(&argument, "clog=")) { g_params.chainLog = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; }
-                if (longCommandWArg(&argument, "hashLog=") || longCommandWArg(&argument, "hlog=")) { g_params.hashLog = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; }
-                if (longCommandWArg(&argument, "searchLog=") || longCommandWArg(&argument, "slog=")) { g_params.searchLog = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; }
-                if (longCommandWArg(&argument, "searchLength=") || longCommandWArg(&argument, "slen=")) { g_params.searchLength = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; }
-                if (longCommandWArg(&argument, "targetLength=") || longCommandWArg(&argument, "tlen=")) { g_params.targetLength = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; }
-                if (longCommandWArg(&argument, "strategy=") || longCommandWArg(&argument, "strat=")) { g_params.strategy = (ZSTD_strategy)(readU32FromChar(&argument)); if (argument[0]==',') { argument++; continue; } else break; }
                 if (longCommandWArg(&argument, "level=") || longCommandWArg(&argument, "lvl=")) { g_params = ZSTD_getCParams(readU32FromChar(&argument), g_blockSize, 0); if (argument[0]==',') { argument++; continue; } else break; }
                 DISPLAY("invalid compression parameter \n");
                 return 1;
@@ -868,6 +2190,7 @@ int main(int argc, const char** argv)
                 DISPLAY("invalid --zstd= format\n");
                 return 1; /* check the end of string */
             }
+            continue;
             /* if not return, success */
         } else if (argument[0]=='-') {
             argument++;
@@ -897,12 +2220,6 @@ int main(int argc, const char** argv)
                     }
                     break;
 
-                case 'O':
-                    argument++;
-                    optimizer = 1;
-                    targetSpeed = readU32FromChar(&argument);
-                    break;
-
                     /* Run Single conf */
                 case 'S':
                     g_singleRun = 1;
@@ -940,7 +2257,9 @@ int main(int argc, const char** argv)
                             g_params.strategy = (ZSTD_strategy)readU32FromChar(&argument);
                             continue;
                         case 'L':
-                            {   int const cLevel = readU32FromChar(&argument);
+                            {   int cLevel;
+                                argument++;
+                                cLevel = readU32FromChar(&argument);
                                 g_params = ZSTD_getCParams(cLevel, g_blockSize, 0);
                                 continue;
                             }
@@ -970,6 +2289,18 @@ int main(int argc, const char** argv)
                     g_grillDuration_s = (double)readU32FromChar(&argument);
                     break;
 
+                /* load dictionary file (only applicable for optimizer rn) */
+                case 'D':
+                    if(i == argc - 1) { /* last argument, return error. */ 
+                        DISPLAY("Dictionary file expected but not given : %d\n", i);
+                        return 1;
+                    } else {
+                        i++;
+                        dictFileName = argv[i];
+                        argument += strlen(argument);
+                    }
+                    break;
+
                     /* Unknown command */
                 default : return badusage(exename);
                 }
@@ -980,7 +2311,6 @@ int main(int argc, const char** argv)
         /* first provided filename is input */
         if (!input_filename) { input_filename=argument; filenamesStart=i; continue; }
     }
-
     if (filenamesStart==0) {
         if (optimizer) {
             DISPLAY("Optimizer Expects File\n");
@@ -990,7 +2320,7 @@ int main(int argc, const char** argv)
         }
     } else {
         if (optimizer) {
-            result = optimizeForSize(input_filename, targetSpeed);
+            result = optimizeForSize(argv+filenamesStart, argc-filenamesStart, dictFileName, target, paramTarget, optimizerCLevel);
         } else {
             result = benchFiles(argv+filenamesStart, argc-filenamesStart);
     }   }