Update fulltable to use same interface

Add seperateFiles flag

tests/README.md

@@ -125,6 +125,7 @@ Full list of arguments
  -t#          : Caps runtime of operation in seconds (default : 99999 seconds (about 27 hours )) 
  -v           : Prints Benchmarking output
  -D           : Next argument dictionary file
+ -s           : Benchmark all files separately
 
 ```
  Any inputs afterwards are treated as files to benchmark.

tests/paramgrill.c

@@ -96,7 +96,7 @@ typedef enum {
 static const int rangetable[NUM_PARAMS] = { WLOG_RANGE, CLOG_RANGE, HLOG_RANGE, SLOG_RANGE, SLEN_RANGE, TLEN_RANGE };
 static const U32 tlen_table[TLEN_RANGE] = { 0, 1, 2, 4, 6, 8, 12, 16, 24, 32, 48, 64, 96, 128, 256, 512, 999 };
 /*-************************************
-*  Benchmark Parameters
+*  Benchmark Parameters/Global Variables
 **************************************/
 
 typedef BYTE U8;
@@ -110,7 +110,7 @@ static U32 g_singleRun = 0;
 static U32 g_optimizer = 0;
 static U32 g_target = 0;
 static U32 g_noSeed = 0;
-static ZSTD_compressionParameters g_params = { 0, 0, 0, 0, 0, 0, ZSTD_greedy };
+static ZSTD_compressionParameters g_params;
 static UTIL_time_t g_time; /* to be used to compare solution finding speeds to compare to original */
 
 
@@ -150,6 +150,14 @@ void BMK_SetNbIterations(int nbLoops)
     DISPLAY("- %u iterations -\n", g_nbIterations);
 }
 
+/*
+ * Additional Global Variables (Defined Above Use)
+ * g_stratName
+ * g_level_constraint
+ * g_alreadyTested
+ * g_maxTries
+ */
+
 /*-*******************************************************
 *  Private functions
 *********************************************************/
@@ -335,17 +343,6 @@ const char* g_stratName[ZSTD_btultra+1] = {
                 "ZSTD_greedy  ", "ZSTD_lazy    ", "ZSTD_lazy2   ",
                 "ZSTD_btlazy2 ", "ZSTD_btopt   ", "ZSTD_btultra "};
 
-/* benchParam but only takes in one input buffer. */
-static int
-BMK_benchParam1(BMK_result_t* resultPtr,
-               const void* srcBuffer, size_t srcSize,
-               const ZSTD_compressionParameters cParams) {
-
-    BMK_return_t res = BMK_benchMem(srcBuffer,srcSize, &srcSize, 1, 0, &cParams, NULL, 0, 0, "File");
-    *resultPtr = res.result;
-    return res.error;
-}
-
 static ZSTD_compressionParameters emptyParams(void) {
     ZSTD_compressionParameters p = { 0, 0, 0, 0, 0, 0, (ZSTD_strategy)0 };
     return p;
@@ -381,14 +378,6 @@ typedef struct {
     ZSTD_DCtx* dctx;
 } contexts_t;
 
-static int BMK_benchParam(BMK_result_t* resultPtr,
-                const buffers_t buf, const contexts_t ctx,
-                const ZSTD_compressionParameters cParams) {
-    BMK_return_t res = BMK_benchMem(buf.srcPtrs[0], buf.srcSize, buf.srcSizes, (unsigned)buf.nbBlocks, 0, &cParams, ctx.dictBuffer, ctx.dictSize, 0, "Files");
-    *resultPtr = res.result;
-    return res.error;
-}
-
 /*-*******************************************************
 *  From Paramgrill
 *********************************************************/
@@ -542,7 +531,7 @@ static int createBuffers(buffers_t* buff, const char* const * const fileNamesTab
     size_t n;
     U64 const totalSizeToLoad = UTIL_getTotalFileSize(fileNamesTable, (U32)nbFiles);
     size_t benchedSize = MIN(BMK_findMaxMem(totalSizeToLoad * 3) / 3, totalSizeToLoad);
-    const size_t blockSize = g_blockSize ? g_blockSize : totalSizeToLoad; //(largest fileSize or total fileSize)
+    const size_t blockSize = g_blockSize ? g_blockSize : totalSizeToLoad;
     U32 const maxNbBlocks = (U32) ((totalSizeToLoad + (blockSize-1)) / blockSize) + (U32)nbFiles;
     U32 blockNb = 0;
 
@@ -830,6 +819,14 @@ static BMK_return_t BMK_benchMemInvertible(const buffers_t buf, const contexts_t
     return results;
 }
 
+static int BMK_benchParam(BMK_result_t* resultPtr,
+                buffers_t buf, contexts_t ctx,
+                const ZSTD_compressionParameters cParams) {
+    BMK_return_t res = BMK_benchMemInvertible(buf, ctx, 0, &cParams, BMK_both, BMK_timeMode, 3);
+    *resultPtr = res.result;
+    return res.error;
+}
+
 /* comparison function: */
 /* strictly better, strictly worse, equal, speed-side adv, size-side adv */
 //Maybe use compress_only for benchmark first run?
@@ -911,7 +908,7 @@ static int insertWinner(winnerInfo_t w, constraint_t targetConstraints) {
 
     }
 
-    //assert(cur_node->next == NULL)
+    assert(cur_node->next == NULL);
     switch(speedSizeCompare(r, cur_node->res.result)) {
         case BETTER_RESULT:
         {
@@ -995,7 +992,7 @@ static void BMK_printWinnerOpt(FILE* f, const U32 cLevel, const BMK_result_t res
         }
     }  
 
-    //prints out tradeoff table if using lvl
+    //prints out tradeoff table if using lvloptimize
     if(g_optmode && g_optimizer) {
         winnerInfo_t w;
         winner_ll_node* n;
@@ -1042,12 +1039,6 @@ static void BMK_printWinnerOpt(FILE* f, const U32 cLevel, const BMK_result_t res
             (double)srcSize / result.cSize, (double)result.cSpeed / (1 MB), (double)result.dSpeed / (1 MB));
 
     }
-
-#if 0
-    if(BMK_timeSpan(g_time) > g_grillDuration_s) {
-        exit(0);
-    }
-#endif
 }
 
 static void BMK_printWinners2(FILE* f, const winnerInfo_t* winners, size_t srcSize)
@@ -1100,13 +1091,13 @@ static void BMK_init_level_constraints(int bytePerSec_level1)
 }
 
 static int BMK_seed(winnerInfo_t* winners, const ZSTD_compressionParameters params, 
-              const void* srcBuffer, size_t srcSize)
+                    buffers_t buf, contexts_t ctx)
 {
     BMK_result_t testResult;
     int better = 0;
     int cLevel;
 
-    BMK_benchParam1(&testResult, srcBuffer, srcSize, params);
+    BMK_benchParam(&testResult, buf, ctx, params);
 
 
     for (cLevel = 1; cLevel <= NB_LEVELS_TRACKED; cLevel++) {
@@ -1122,15 +1113,15 @@ static int BMK_seed(winnerInfo_t* winners, const ZSTD_compressionParameters para
             /* first solution for this cLevel */
             winners[cLevel].result = testResult;
             winners[cLevel].params = params;
-            BMK_printWinner(stdout, cLevel, testResult, params, srcSize);
+            BMK_printWinner(stdout, cLevel, testResult, params, buf.srcSize);
             better = 1;
             continue;
         }
 
         if ((double)testResult.cSize <= ((double)winners[cLevel].result.cSize * (1. + (0.02 / cLevel))) ) {
             /* Validate solution is "good enough" */
-            double W_ratio = (double)srcSize / testResult.cSize;
+            double W_ratio = (double)buf.srcSize / testResult.cSize;
-            double O_ratio = (double)srcSize / winners[cLevel].result.cSize;
+            double O_ratio = (double)buf.srcSize / winners[cLevel].result.cSize;
             double W_ratioNote = log (W_ratio);
             double O_ratioNote = log (O_ratio);
             size_t W_DMemUsed = (1 << params.windowLog) + (16 KB);
@@ -1187,7 +1178,7 @@ static int BMK_seed(winnerInfo_t* winners, const ZSTD_compressionParameters para
 
             winners[cLevel].result = testResult;
             winners[cLevel].params = params;
-            BMK_printWinner(stdout, cLevel, testResult, params, srcSize);
+            BMK_printWinner(stdout, cLevel, testResult, params, buf.srcSize);
 
             better = 1;
     }   }
@@ -1315,6 +1306,26 @@ static void paramVariation(ZSTD_compressionParameters* ptr, const varInds_t* var
     *ptr = p;
 }
 
+/* maybe put strategy back in */
+static void paramVariationWithStrategy(ZSTD_compressionParameters* ptr, const varInds_t* varyParams, const int varyLen, const U32 nbChanges)
+{
+    ZSTD_compressionParameters p;
+    U32 validated = 0;
+    while (!validated) {
+        U32 i;
+        p = *ptr;
+        for (i = 0 ; i < nbChanges ; i++) {
+            const U32 changeID = FUZ_rand(&g_rand) % ((varyLen + 1) << 1);
+            if(changeID < (U32)(varyLen << 1)) {
+                paramVaryOnce(varyParams[changeID >> 1], ((changeID & 1) << 1) - 1, &p);
+            } else {
+                p.strategy += ((FUZ_rand(&g_rand) % 2) << 1) - 1; /* +/- 1 */
+            }
+        }
+        validated = !ZSTD_isError(ZSTD_checkCParams(p));
+    }
+    *ptr = p;
+}
 /* length of memo table given free variables */
 static size_t memoTableLen(const varInds_t* varyParams, const int varyLen) {
     size_t arrayLen = 1;
@@ -1463,6 +1474,17 @@ static U8** createMemoTableArray(ZSTD_compressionParameters paramConstraints, co
     return mtAll;
 }
 
+static ZSTD_compressionParameters maskParams(ZSTD_compressionParameters base, ZSTD_compressionParameters mask) {
+    base.windowLog = mask.windowLog ? mask.windowLog : base.windowLog;
+    base.chainLog = mask.chainLog ? mask.chainLog : base.chainLog;
+    base.hashLog = mask.hashLog ? mask.hashLog : base.hashLog;
+    base.searchLog = mask.searchLog ? mask.searchLog : base.searchLog;
+    base.searchLength = mask.searchLength ? mask.searchLength : base.searchLength;
+    base.targetLength = mask.targetLength ? mask.targetLength : base.targetLength;
+    base.strategy = mask.strategy ? mask.strategy : base.strategy;
+    return base;
+}
+
 #define PARAMTABLELOG   25
 #define PARAMTABLESIZE (1<<PARAMTABLELOG)
 #define PARAMTABLEMASK (PARAMTABLESIZE-1)
@@ -1479,7 +1501,7 @@ static BYTE* NB_TESTS_PLAYED(ZSTD_compressionParameters p) {
 
 static void playAround(FILE* f, winnerInfo_t* winners,
                        ZSTD_compressionParameters params,
-                       const void* srcBuffer, size_t srcSize)
+                       buffers_t buf, contexts_t ctx)
 {
     int nbVariations = 0;
     UTIL_time_t const clockStart = UTIL_getTime();
@@ -1491,7 +1513,7 @@ static void playAround(FILE* f, winnerInfo_t* winners,
         BYTE* b;
 
         if (nbVariations++ > g_maxNbVariations) break;
-        paramVariation(&p, unconstrained, 7, 4);
+        paramVariationWithStrategy(&p, unconstrained, NUM_PARAMS, 4);
 
         /* exclude faster if already played params */
         if (FUZ_rand(&g_rand) & ((1 << *NB_TESTS_PLAYED(p))-1))
@@ -1500,11 +1522,11 @@ static void playAround(FILE* f, winnerInfo_t* winners,
         /* test */
         b = NB_TESTS_PLAYED(p);
         (*b)++;
-        if (!BMK_seed(winners, p, srcBuffer, srcSize)) continue;
+        if (!BMK_seed(winners, p, buf, ctx)) continue;
 
         /* improvement found => search more */
-        BMK_printWinners(f, winners, srcSize);
+        BMK_printWinners(f, winners, buf.srcSize);
-        playAround(f, winners, p, srcBuffer, srcSize);
+        playAround(f, winners, p, buf, ctx);
     }
 
 }
@@ -1551,35 +1573,24 @@ static void randomConstrainedParams(ZSTD_compressionParameters* pc, varInds_t* v
 
 static void BMK_selectRandomStart(
                        FILE* f, winnerInfo_t* winners,
-                       const void* srcBuffer, size_t srcSize)
+                       buffers_t buf, contexts_t ctx)
 {
     U32 const id = FUZ_rand(&g_rand) % (NB_LEVELS_TRACKED+1);
     if ((id==0) || (winners[id].params.windowLog==0)) {
         /* use some random entry */
-        ZSTD_compressionParameters const p = ZSTD_adjustCParams(randomParams(), srcSize, 0);
+        ZSTD_compressionParameters const p = ZSTD_adjustCParams(randomParams(), buf.srcSize, 0);
-        playAround(f, winners, p, srcBuffer, srcSize);
+        playAround(f, winners, p, buf, ctx);
     } else {
-        playAround(f, winners, winners[id].params, srcBuffer, srcSize);
+        playAround(f, winners, winners[id].params, buf, ctx);
     }
 }
 
-static void BMK_benchOnce(const void* srcBuffer, size_t srcSize)
+static void BMK_benchFullTable(buffers_t buf, contexts_t ctx, const size_t maxBlockSize) 
-{
-    BMK_result_t testResult;
-    g_params = ZSTD_adjustCParams(g_params, srcSize, 0);
-    BMK_benchParam1(&testResult, srcBuffer, srcSize, g_params);
-    DISPLAY("Compression Ratio: %.3f  Compress Speed: %.1f MB/s Decompress Speed: %.1f MB/s\n", (double)srcSize / testResult.cSize, 
-        (double)testResult.cSpeed / 1000000, (double)testResult.dSpeed / 1000000);
-    return;
-}
-
-static void BMK_benchFullTable(const void* srcBuffer, size_t srcSize)
 {
     ZSTD_compressionParameters params;
     winnerInfo_t winners[NB_LEVELS_TRACKED+1];
     const char* const rfName = "grillResults.txt";
     FILE* const f = fopen(rfName, "w");
-    const size_t blockSize = g_blockSize ? g_blockSize : srcSize;   /* cut by block or not ? */
 
     /* init */
     assert(g_singleRun==0);
@@ -1590,9 +1601,9 @@ static void BMK_benchFullTable(const void* srcBuffer, size_t srcSize)
         BMK_init_level_constraints(g_target * (1 MB));
     } else {
         /* baseline config for level 1 */
-        ZSTD_compressionParameters const l1params = ZSTD_getCParams(1, blockSize, 0);
+        ZSTD_compressionParameters const l1params = ZSTD_getCParams(1, maxBlockSize, ctx.dictSize); //is dictionary ever even useful here? 
         BMK_result_t testResult;
-        BMK_benchParam1(&testResult, srcBuffer, srcSize, l1params);
+        BMK_benchParam(&testResult, buf, ctx, l1params);
         BMK_init_level_constraints((int)((testResult.cSpeed * 31) / 32));
     }
 
@@ -1600,61 +1611,124 @@ static void BMK_benchFullTable(const void* srcBuffer, size_t srcSize)
     {   const int maxSeeds = g_noSeed ? 1 : ZSTD_maxCLevel();
         int i;
         for (i=0; i<=maxSeeds; i++) {
-            params = ZSTD_getCParams(i, blockSize, 0);
+            params = ZSTD_getCParams(i, maxBlockSize, 0);
-            BMK_seed(winners, params, srcBuffer, srcSize);
+            BMK_seed(winners, params, buf, ctx);
     }   }
-    BMK_printWinners(f, winners, srcSize);
+    BMK_printWinners(f, winners, buf.srcSize);
 
     /* start tests */
     {   const time_t grillStart = time(NULL);
         do {
-            BMK_selectRandomStart(f, winners, srcBuffer, srcSize);
+            BMK_selectRandomStart(f, winners, buf, ctx);
         } while (BMK_timeSpan(grillStart) < g_grillDuration_s);
     }
 
     /* end summary */
-    BMK_printWinners(f, winners, srcSize);
+    BMK_printWinners(f, winners, buf.srcSize);
     DISPLAY("grillParams operations completed \n");
 
     /* clean up*/
     fclose(f);
 }
 
-static void BMK_benchMemInit(const void* srcBuffer, size_t srcSize)
-{
-    if (g_singleRun)
-        return BMK_benchOnce(srcBuffer, srcSize);
-    else
-        return BMK_benchFullTable(srcBuffer, srcSize);
-}
-
-
 static int benchSample(void)
 {
     const char* const name = "Sample 10MB";
     size_t const benchedSize = 10 MB;
+    U32 blockSize = g_blockSize ? g_blockSize : benchedSize;
+    U32 const maxNbBlocks = (U32) ((benchedSize + (blockSize-1)) / blockSize) + 1;
+    size_t splitSize = 0;
 
-    void* origBuff = malloc(benchedSize);
+    buffers_t buf;
-    if (!origBuff) { perror("not enough memory"); return 12; }
+    contexts_t ctx;
 
-    /* Fill buffer */
+    buf.srcPtrs = (const void**)calloc(maxNbBlocks, sizeof(void*));
-    RDG_genBuffer(origBuff, benchedSize, g_compressibility, 0.0, 0);
+    buf.dstPtrs = (void**)calloc(maxNbBlocks, sizeof(void*));
+    buf.resPtrs = (void**)calloc(maxNbBlocks, sizeof(void*));
+    buf.srcSizes = (size_t*)malloc(maxNbBlocks * sizeof(size_t));
+    buf.dstSizes = (size_t*)malloc(maxNbBlocks * sizeof(size_t));
+    buf.dstCapacities = (size_t*)malloc(maxNbBlocks * sizeof(size_t));
+    buf.resSizes = (size_t*)malloc(maxNbBlocks * sizeof(size_t));
+    buf.srcSize = benchedSize;
+
+    if(!buf.srcPtrs || !buf.dstPtrs || !buf.resPtrs || !buf.srcSizes || !buf.dstSizes || !buf.dstCapacities || !buf.resSizes) {
+        DISPLAY("Allocation Error\n");
+        freeBuffers(buf);
+        return 1;
+    } 
+
+    buf.srcBuffer = malloc(benchedSize);
+    buf.srcPtrs[0] = (const void*)buf.srcBuffer;
+    buf.dstPtrs[0] = malloc(ZSTD_compressBound(benchedSize) + 1024 * maxNbBlocks);
+    buf.resPtrs[0] = malloc(benchedSize);
+
+    if(!buf.srcPtrs[0] || !buf.dstPtrs[0] || !buf.resPtrs[0]) {
+        DISPLAY("Allocation Error\n");
+        freeBuffers(buf);
+        return 1;
+    }
+
+
+    splitSize = MIN(benchedSize, blockSize);
+    buf.srcSizes[0] = splitSize;
+    buf.dstCapacities[0] = ZSTD_compressBound(splitSize);
+    buf.resSizes[0] = splitSize;
+
+    for(buf.nbBlocks = 1; splitSize < benchedSize; buf.nbBlocks++) {
+        const size_t i = buf.nbBlocks;
+        const size_t nextBlockSize = MIN(benchedSize - splitSize, blockSize);
+        buf.srcSizes[i] = nextBlockSize;
+        buf.dstCapacities[i] = ZSTD_compressBound(nextBlockSize);
+        buf.resSizes[i] = nextBlockSize;
+        buf.srcPtrs[i] = (const void*)(((const char*)buf.srcPtrs[i-1]) + buf.srcSizes[i-1]);
+        buf.dstPtrs[i] = (void*)(((char*)buf.dstPtrs[i-1]) + buf.dstSizes[i-1]);
+        buf.resPtrs[i] = (void*)(((char*)buf.resPtrs[i-1]) + buf.resSizes[i-1]);
+        splitSize += nextBlockSize;
+    }
+
+    if(createContexts(&ctx, NULL)) {
+        DISPLAY("Context Creation Error\n");
+        freeBuffers(buf);
+        return 1;
+    }
+
+    RDG_genBuffer(buf.srcBuffer, benchedSize, g_compressibility, 0.0, 0);
 
     /* bench */
     DISPLAY("\r%79s\r", "");
     DISPLAY("using %s %i%%: \n", name, (int)(g_compressibility*100));
-    BMK_benchMemInit(origBuff, benchedSize);
 
-    free(origBuff);
+    BMK_benchFullTable(buf, ctx, MIN(blockSize, benchedSize));
+
+    freeBuffers(buf);
+    freeContexts(ctx);
+
     return 0;
 }
 
 
-static int benchOnce(const char** fileNamesTable, int nbFiles, const char* dictFileName) {
+static int benchOnce(buffers_t buf, contexts_t ctx) {
+    BMK_result_t testResult;
+
+    if(BMK_benchParam(&testResult, buf, ctx, g_params)) {
+        DISPLAY("Error during benchmarking\n");
+        return 1;
+    }
+
+    DISPLAY("Compression Ratio: %.3f  Compress Speed: %.1f MB/s Decompress Speed: %.1f MB/s\n", (double)buf.srcSize / testResult.cSize, 
+        (double)testResult.cSpeed / (1 MB), (double)testResult.dSpeed / (1 MB));
+    return 0;
+}
+
+/* benchFiles() :
+ * note: while this function takes a table of filenames,
+ * in practice, only the first filename will be used */
+int benchFiles(const char** fileNamesTable, int nbFiles, const char* dictFileName, int cLevel)
+{
     buffers_t buf;
     contexts_t ctx;
-    BMK_result_t testResult;
     size_t maxBlockSize = 0, i;
+    int ret = 0;
 
     if(createBuffers(&buf, fileNamesTable, nbFiles)) {
         DISPLAY("unable to load files\n");
@@ -1671,86 +1745,24 @@ static int benchOnce(const char** fileNamesTable, int nbFiles, const char* dictF
         maxBlockSize = MAX(maxBlockSize, buf.srcSizes[i]);
     }
 
-    g_params = ZSTD_adjustCParams(g_params, maxBlockSize, 0);
+    DISPLAY("\r%79s\r", "");
-
+    if(nbFiles == 1) {
-    if(BMK_benchParam(&testResult, buf, ctx, g_params)) {
+        DISPLAY("using %s : \n", fileNamesTable[0]);
-        DISPLAY("Error during benchmarking\n");
+    } else {
-        freeBuffers(buf);
+        DISPLAY("using %d Files : \n", nbFiles);
-        freeContexts(ctx);
-        return 3;
     }
 
-    DISPLAY("Compression Ratio: %.3f  Compress Speed: %.1f MB/s Decompress Speed: %.1f MB/s\n", (double)buf.srcSize / testResult.cSize, 
+    g_params = ZSTD_adjustCParams(maskParams(ZSTD_getCParams(cLevel, maxBlockSize, ctx.dictSize), g_params), maxBlockSize, ctx.dictSize);
-        (double)testResult.cSpeed / (1 MB), (double)testResult.dSpeed / (1 MB));
+
+    if(g_singleRun) {
+        ret = benchOnce(buf, ctx);
+    } else {
+        BMK_benchFullTable(buf, ctx, maxBlockSize);
+    }
 
     freeBuffers(buf);
     freeContexts(ctx);
-    return 0;
+    return ret;
-}
-
-/* benchFiles() :
- * note: while this function takes a table of filenames,
- * in practice, only the first filename will be used */
-//TODO: dictionaries still not supported in fullTable mode
-int benchFiles(const char** fileNamesTable, int nbFiles, const char* dictFileName)
-{
-    int fileIdx=0;
-
-    if(g_singleRun) {
-        return benchOnce(fileNamesTable, nbFiles, dictFileName);
-    }
-
-    /* Loop for each file */
-    while (fileIdx<nbFiles) {
-        const char* const inFileName = fileNamesTable[fileIdx++];
-        FILE* const inFile = fopen( inFileName, "rb" );
-        U64 const inFileSize = UTIL_getFileSize(inFileName);
-        size_t benchedSize;
-        void* origBuff;
-
-        /* Check file existence */
-        if (inFile==NULL) {
-            DISPLAY( "Pb opening %s\n", inFileName);
-            return 11;
-        }
-        if (inFileSize == UTIL_FILESIZE_UNKNOWN) {
-            DISPLAY("Pb evaluating size of %s \n", inFileName);
-            fclose(inFile);
-            return 11;
-        }
-
-        /* Memory allocation */
-        benchedSize = BMK_findMaxMem(inFileSize*3) / 3;
-        if ((U64)benchedSize > inFileSize) benchedSize = (size_t)inFileSize;
-        if (benchedSize < inFileSize)
-            DISPLAY("Not enough memory for '%s' full size; testing %i MB only...\n", inFileName, (int)(benchedSize>>20));
-        origBuff = malloc(benchedSize);
-        if (origBuff==NULL) {
-            DISPLAY("\nError: not enough memory!\n");
-            fclose(inFile);
-            return 12;
-        }
-
-        /* Fill input buffer */
-        DISPLAY("Loading %s...       \r", inFileName);
-        {   size_t const readSize = fread(origBuff, 1, benchedSize, inFile);
-            fclose(inFile);
-            if(readSize != benchedSize) {
-                DISPLAY("\nError: problem reading file '%s' !!    \n", inFileName);
-                free(origBuff);
-                return 13;
-        }   }
-
-        /* bench */
-        DISPLAY("\r%79s\r", "");
-        DISPLAY("using %s : \n", inFileName);
-        BMK_benchMemInit(origBuff, benchedSize);
-
-        /* clean */
-        free(origBuff);
-    }
-
-    return 0;
 }
 
 /* Benchmarking which stops when we are sufficiently sure the solution is infeasible / worse than the winner */
@@ -2074,17 +2086,6 @@ static int nextStrategy(const int currentStrategy, const int bestStrategy) {
     }
 }
 
-static ZSTD_compressionParameters maskParams(ZSTD_compressionParameters base, ZSTD_compressionParameters mask) {
-    base.windowLog = mask.windowLog ? mask.windowLog : base.windowLog;
-    base.chainLog = mask.chainLog ? mask.chainLog : base.chainLog;
-    base.hashLog = mask.hashLog ? mask.hashLog : base.hashLog;
-    base.searchLog = mask.searchLog ? mask.searchLog : base.searchLog;
-    base.searchLength = mask.searchLength ? mask.searchLength : base.searchLength;
-    base.targetLength = mask.targetLength ? mask.targetLength : base.targetLength;
-    base.strategy = mask.strategy ? mask.strategy : base.strategy;
-    return base;
-}
-
 /* experiment with playing with this and decay value */
 
 /* main fn called when using --optimize */
@@ -2115,6 +2116,8 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_
     contexts_t ctx;
     buffers_t buf;
 
+    g_time = UTIL_getTime();
+
     /* Init */
     if(!cParamValid(paramTarget)) {
         return 1;
@@ -2202,7 +2205,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_
         g_lvltarget.cSize /= ((double)g_strictness) / 100;
 
         target.cSpeed = (U32)g_lvltarget.cSpeed;  
-        target.dSpeed = (U32)g_lvltarget.dSpeed; //See if this is worth
+        target.dSpeed = (U32)g_lvltarget.dSpeed; //See if this is reasonable. 
 
         BMK_printWinnerOpt(stdout, cLevel, winner.result, winner.params, target, buf.srcSize);
     }
@@ -2214,6 +2217,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_
     } else {
         DISPLAY("optimizing for %lu Files", (unsigned long)nbFiles);
     }
+
     if(target.cSpeed != 0) { DISPLAY(" - limit compression speed %u MB/s", target.cSpeed >> 20); }
     if(target.dSpeed != 0) { DISPLAY(" - limit decompression speed %u MB/s", target.dSpeed >> 20); }
     if(target.cMem != (U32)-1) { DISPLAY(" - limit memory %u MB", target.cMem >> 20); }
@@ -2369,6 +2373,7 @@ static int usage_advanced(void)
     DISPLAY( " -t#          : Caps runtime of operation in seconds (default : %u seconds (%.1f hours)) \n", (U32)g_grillDuration_s, g_grillDuration_s / 3600);
     DISPLAY( " -v           : Prints Benchmarking output\n");
     DISPLAY( " -D           : Next argument dictionary file\n");
+    DISPLAY( " -s           : Seperate Files\n");
     return 0;
 }
 
@@ -2401,13 +2406,13 @@ int main(int argc, const char** argv)
     const char* input_filename = NULL;
     const char* dictFileName = NULL;
     U32 main_pause = 0;
-    int optimizerCLevel = 0;
+    int cLevel = 0;
+    int seperateFiles = 0;
 
     constraint_t target = { 0, 0, (U32)-1 }; 
 
-    ZSTD_compressionParameters paramTarget = { 0, 0, 0, 0, 0, 0, 0 };
+    ZSTD_compressionParameters paramTarget = emptyParams();
-
+    g_params = emptyParams();
-    g_time = UTIL_getTime();
 
     assert(argc>=1);   /* for exename */
 
@@ -2430,11 +2435,11 @@ int main(int argc, const char** argv)
                 PARSE_SUB_ARGS("compressionSpeed=" ,  "cSpeed=", target.cSpeed);
                 PARSE_SUB_ARGS("decompressionSpeed=", "dSpeed=", target.dSpeed); 
                 PARSE_SUB_ARGS("compressionMemory=" , "cMem=", target.cMem);
-                PARSE_SUB_ARGS("level=", "lvl=", optimizerCLevel);
+                PARSE_SUB_ARGS("level=", "lvl=", cLevel);
                 PARSE_SUB_ARGS("strict=", "stc=", g_strictness);
                 PARSE_SUB_ARGS("preferSpeed=", "prfSpd=", g_speedMultiplier);
                 PARSE_SUB_ARGS("preferRatio=", "prfRto=", g_ratioMultiplier);
-                PARSE_SUB_ARGS("maxTries", "tries", g_maxTries);
+                PARSE_SUB_ARGS("maxTries=", "tries=", g_maxTries);
 
                 DISPLAY("invalid optimization parameter \n");
                 return 1;
@@ -2448,10 +2453,11 @@ int main(int argc, const char** argv)
         } else if (longCommandWArg(&argument, "--zstd=")) {
         /* Decode command (note : aggregated commands are allowed) */
             g_singleRun = 1;
-            g_params = ZSTD_getCParams(2, g_blockSize, 0);
+            cLevel = 2;
             for ( ; ;) {
                 PARSE_CPARAMS(g_params)
-                if (longCommandWArg(&argument, "level=") || longCommandWArg(&argument, "lvl=")) { g_params = ZSTD_getCParams(readU32FromChar(&argument), g_blockSize, 0); if (argument[0]==',') { argument++; continue; } else break; }
+                if (longCommandWArg(&argument, "level=") || longCommandWArg(&argument, "lvl=")) { cLevel = readU32FromChar(&argument); g_params = emptyParams(); if (argument[0]==',') { argument++; continue; } else break; }
+
                 DISPLAY("invalid compression parameter \n");
                 return 1;
             }
@@ -2528,8 +2534,8 @@ int main(int argc, const char** argv)
                             continue;
                         case 'L':
                             {   argument++;
-                                int const cLevel = readU32FromChar(&argument);
+                                cLevel = readU32FromChar(&argument);
-                                g_params = ZSTD_getCParams(cLevel, g_blockSize, 0);
+                                g_params = emptyParams();
                                 continue;
                             }
                         default : ;
@@ -2558,6 +2564,10 @@ int main(int argc, const char** argv)
                     g_grillDuration_s = (double)readU32FromChar(&argument);
                     break;
 
+                case 's':
+                    seperateFiles = 1;
+                    break;
+
                 /* load dictionary file (only applicable for optimizer rn) */
                 case 'D':
                     if(i == argc - 1) { /* last argument, return error. */ 
@@ -2588,11 +2598,24 @@ int main(int argc, const char** argv)
             result = benchSample();
         }
     } else {
-        if (g_optimizer) {
+        if(seperateFiles) {
-            result = optimizeForSize(argv+filenamesStart, argc-filenamesStart, dictFileName, target, paramTarget, optimizerCLevel);
+            for(i = 0; i < argc - filenamesStart; i++) {
+                if (g_optimizer) {
+                    result = optimizeForSize(argv+filenamesStart + i, 1, dictFileName, target, paramTarget, cLevel);
+                    if(result) { DISPLAY("Error on File %d", i); return result; }
+                } else {
+                    result = benchFiles(argv+filenamesStart + i, 1, dictFileName, cLevel);
+                    if(result) { DISPLAY("Error on File %d", i); return result; }
+                }
+            }
         } else {
-            result = benchFiles(argv+filenamesStart, argc-filenamesStart, dictFileName);
+            if (g_optimizer) {
-    }   }
+                result = optimizeForSize(argv+filenamesStart, argc-filenamesStart, dictFileName, target, paramTarget, cLevel);
+            } else {
+                result = benchFiles(argv+filenamesStart, argc-filenamesStart, dictFileName, cLevel);
+            }
+        }   
+    }
 
     if (main_pause) { int unused; printf("press enter...\n"); unused = getchar(); (void)unused; }