Merge branch 'travisTest' of github.com:facebook/zstd into travisTest

.travis.yml

@@ -10,8 +10,13 @@ addons:
 matrix:
   include:
     # Ubuntu 14.04
+<<<<<<< HEAD
     - env: Cmd='make gcc6install && CC=gcc-6 make -j all
              && make clean && CC=gcc-6 make clean uasan-test-zstd'
+=======
+    - env: Cmd='make test'
+    - env: Cmd='make gcc6install && CC=gcc-6 make -j all && make clean && CC=gcc-6 make clean uasan-test-zstd </dev/null' # also test when stdin is not a tty
+>>>>>>> 0840d02ecf74eae656e0df0d900000d9b0154cde
     - env: Cmd='make gcc6install libc6install && CC=gcc-6 make clean uasan-test-zstd32'
     - env: Cmd='make gcc7install && CC=gcc-7 make clean uasan-test-zstd'
     - env: Cmd='make clang38install && CC=clang-3.8 make clean msan-test-zstd'

Makefile

@@ -63,8 +63,9 @@ zlibwrapper:
 	$(MAKE) -C $(ZWRAPDIR) test
 
 .PHONY: test
+test: MOREFLAGS += -g -DDEBUGLEVEL=1 -Werror
 test:
-	$(MAKE) -C $(PRGDIR) allVariants MOREFLAGS+="-g -DDEBUGLEVEL=1"
+	MOREFLAGS="$(MOREFLAGS)" $(MAKE) -C $(PRGDIR) allVariants
 	$(MAKE) -C $(TESTDIR) $@
 
 .PHONY: shortest

build/VS2010/fullbench-dll/fullbench-dll.vcxproj

@@ -167,11 +167,13 @@
   <ItemGroup>
     <ClCompile Include="..\..\..\lib\common\xxhash.c" />
     <ClCompile Include="..\..\..\programs\datagen.c" />
+    <ClCompile Include="..\..\..\programs\bench.c" />
     <ClCompile Include="..\..\..\tests\fullbench.c" />
   </ItemGroup>
   <ItemGroup>
     <ClInclude Include="..\..\..\lib\zstd.h" />
     <ClInclude Include="..\..\..\programs\datagen.h" />
+    <ClInclude Include="..\..\..\programs\bench.h" />
     <ClInclude Include="..\..\..\programs\util.h" />
   </ItemGroup>
   <ItemGroup>

build/VS2010/fullbench/fullbench.vcxproj

@@ -176,6 +176,7 @@
     <ClCompile Include="..\..\..\lib\decompress\huf_decompress.c" />
     <ClCompile Include="..\..\..\lib\decompress\zstd_decompress.c" />
     <ClCompile Include="..\..\..\programs\datagen.c" />
+    <ClCompile Include="..\..\..\programs\bench.c" />
     <ClCompile Include="..\..\..\tests\fullbench.c" />
   </ItemGroup>
   <ItemGroup>
@@ -197,6 +198,7 @@
     <ClInclude Include="..\..\..\lib\legacy\zstd_legacy.h" />
     <ClInclude Include="..\..\..\programs\datagen.h" />
     <ClInclude Include="..\..\..\programs\util.h" />
+    <ClInclude Include="..\..\..\programs\bench.h" />
   </ItemGroup>
   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
   <ImportGroup Label="ExtensionTargets">

contrib/meson/meson.build

@@ -18,6 +18,7 @@ libzstd_srcs = [
     join_paths(common_dir, 'error_private.c'),
     join_paths(common_dir, 'xxhash.c'),
     join_paths(compress_dir, 'fse_compress.c'),
+    join_paths(compress_dir, 'hist.c'),
     join_paths(compress_dir, 'huf_compress.c'),
     join_paths(compress_dir, 'zstd_compress.c'),
     join_paths(compress_dir, 'zstd_fast.c'),
@@ -130,6 +131,7 @@ test('fuzzer', fuzzer)
 if target_machine.system() != 'windows'
     paramgrill = executable('paramgrill',
                             datagen_c, join_paths(tests_dir, 'paramgrill.c'),
+                            join_paths(programs_dir, 'bench.c'),
                             include_directories: test_includes,
                             link_with: libzstd,
                             dependencies: libm)

contrib/premake/premake4.lua

@@ -0,0 +1,6 @@
+-- Include zstd.lua in your GENie or premake4 file, which exposes a project_zstd function
+dofile('zstd.lua')
+
+solution 'example'
+	configurations { 'Debug', 'Release' }
+	project_zstd('../../lib/')

contrib/premake/zstd.lua

@@ -0,0 +1,80 @@
+-- This GENie/premake file copies the behavior of the Makefile in the lib folder.
+-- Basic usage: project_zstd(ZSTD_DIR)
+
+function project_zstd(dir, compression, decompression, deprecated, dictbuilder, legacy)
+	if compression == nil then compression = true end
+	if decompression == nil then decompression = true end
+	if deprecated == nil then deprecated = false end
+	if dictbuilder == nil then dictbuilder = false end
+
+	if legacy == nil then legacy = 0 end
+
+	if not compression then
+		dictbuilder = false
+		deprecated = false
+	end
+
+	if not decompression then
+		legacy = 0
+		deprecated = false
+	end
+
+	project 'zstd'
+		kind 'StaticLib'
+		language 'C'
+
+		files {
+			dir .. 'zstd.h',
+			dir .. 'common/**.c',
+			dir .. 'common/**.h'
+		}
+
+		if compression then
+			files {
+				dir .. 'compress/**.c',
+				dir .. 'compress/**.h'
+			}
+		end
+
+		if decompression then
+			files {
+				dir .. 'decompress/**.c',
+				dir .. 'decompress/**.h'
+			}
+		end
+
+		if dictbuilder then
+			files {
+				dir .. 'dictBuilder/**.c',
+				dir .. 'dictBuilder/**.h'
+			}
+		end
+
+		if deprecated then
+			files {
+				dir .. 'deprecated/**.c',
+				dir .. 'deprecated/**.h'
+			}
+		end
+
+		if legacy ~= 0 then
+			if legacy >= 8 then
+				files {
+					dir .. 'legacy/zstd_v0' .. (legacy - 7) .. '.*'
+				}
+			end
+			includedirs {
+				dir .. 'legacy'
+			}
+		end
+
+		includedirs {
+			dir,
+			dir .. 'common'
+		}
+
+		defines {
+			'XXH_NAMESPACE=ZSTD_',
+			'ZSTD_LEGACY_SUPPORT=' .. legacy
+		}
+end

doc/zstd_compression_format.md

@@ -488,20 +488,20 @@ For values spanning several bytes, convention is __little-endian__.
 __`Size_Format` for `Raw_Literals_Block` and `RLE_Literals_Block`__ :
 
 `Size_Format` uses 1 _or_ 2 bits.
-Its value is : `Size_Format = (Header[0]>>2) & 3`
+Its value is : `Size_Format = (Literals_Section_Header[0]>>2) & 3`
 
 - `Size_Format` == 00 or 10 : `Size_Format` uses 1 bit.
                `Regenerated_Size` uses 5 bits (0-31).
                `Literals_Section_Header` uses 1 byte.
-               `Regenerated_Size = Header[0]>>3`
+               `Regenerated_Size = Literals_Section_Header[0]>>3`
 - `Size_Format` == 01 : `Size_Format` uses 2 bits.
                `Regenerated_Size` uses 12 bits (0-4095).
                `Literals_Section_Header` uses 2 bytes.
-               `Regenerated_Size = (Header[0]>>4) + (Header[1]<<4)`
+               `Regenerated_Size = (Literals_Section_Header[0]>>4) + (Literals_Section_Header[1]<<4)`
 - `Size_Format` == 11 : `Size_Format` uses 2 bits.
                `Regenerated_Size` uses 20 bits (0-1048575).
                `Literals_Section_Header` uses 3 bytes.
-               `Regenerated_Size = (Header[0]>>4) + (Header[1]<<4) + (Header[2]<<12)`
+               `Regenerated_Size = (Literals_Section_Header[0]>>4) + (Literals_Section_Header[1]<<4) + (Literals_Section_Header[2]<<12)`
 
 Only Stream1 is present for these cases.
 Note : it's allowed to represent a short value (for example `13`)

lib/common/compiler.h

@@ -88,7 +88,11 @@
   #endif
 #endif
 
-/* prefetch */
+/* prefetch
+ * can be disabled, by declaring NO_PREFETCH macro */
+#if defined(NO_PREFETCH)
+#  define PREFETCH(ptr)   /* disabled */
+#else
 #  if defined(_MSC_VER) && (defined(_M_X64) || defined(_M_I86))  /* _mm_prefetch() is not defined outside of x86/x64 */
 #    include <mmintrin.h>   /* https://msdn.microsoft.com/fr-fr/library/84szxsww(v=vs.90).aspx */
 #    define PREFETCH(ptr)   _mm_prefetch((const char*)ptr, _MM_HINT_T0)
@@ -97,6 +101,7 @@
 #  else
 #    define PREFETCH(ptr)   /* disabled */
 #  endif
+#endif  /* NO_PREFETCH */
 
 /* disable warnings */
 #ifdef _MSC_VER    /* Visual Studio */

lib/dictBuilder/cover.c

@@ -39,6 +39,7 @@
 *  Constants
 ***************************************/
 #define COVER_MAX_SAMPLES_SIZE (sizeof(size_t) == 8 ? ((U32)-1) : ((U32)1 GB))
+#define DEFAULT_SPLITPOINT 1.0
 
 /*-*************************************
 *  Console display
@@ -203,6 +204,8 @@ typedef struct {
   size_t *offsets;
   const size_t *samplesSizes;
   size_t nbSamples;
+  size_t nbTrainSamples;
+  size_t nbTestSamples;
   U32 *suffix;
   size_t suffixSize;
   U32 *freqs;
@@ -222,7 +225,7 @@ static COVER_ctx_t *g_ctx = NULL;
  */
 static size_t COVER_sum(const size_t *samplesSizes, unsigned nbSamples) {
   size_t sum = 0;
-  size_t i;
+  unsigned i;
   for (i = 0; i < nbSamples; ++i) {
     sum += samplesSizes[i];
   }
@@ -494,6 +497,10 @@ static int COVER_checkParameters(ZDICT_cover_params_t parameters,
   if (parameters.d > parameters.k) {
     return 0;
   }
+  /* 0 < splitPoint <= 1 */
+  if (parameters.splitPoint <= 0 || parameters.splitPoint > 1){
+    return 0;
+  }
   return 1;
 }
 
@@ -531,9 +538,14 @@ static void COVER_ctx_destroy(COVER_ctx_t *ctx) {
  */
 static int COVER_ctx_init(COVER_ctx_t *ctx, const void *samplesBuffer,
                           const size_t *samplesSizes, unsigned nbSamples,
-                          unsigned d) {
+                          unsigned d, double splitPoint) {
   const BYTE *const samples = (const BYTE *)samplesBuffer;
   const size_t totalSamplesSize = COVER_sum(samplesSizes, nbSamples);
+  /* Split samples into testing and training sets */
+  const unsigned nbTrainSamples = splitPoint < 1.0 ? (unsigned)((double)nbSamples * splitPoint) : nbSamples;
+  const unsigned nbTestSamples = splitPoint < 1.0 ? nbSamples - nbTrainSamples : nbSamples;
+  const size_t trainingSamplesSize = splitPoint < 1.0 ? COVER_sum(samplesSizes, nbTrainSamples) : totalSamplesSize;
+  const size_t testSamplesSize = splitPoint < 1.0 ? COVER_sum(samplesSizes + nbTrainSamples, nbTestSamples) : totalSamplesSize;
   /* Checks */
   if (totalSamplesSize < MAX(d, sizeof(U64)) ||
       totalSamplesSize >= (size_t)COVER_MAX_SAMPLES_SIZE) {
@@ -541,15 +553,29 @@ static int COVER_ctx_init(COVER_ctx_t *ctx, const void *samplesBuffer,
                  (U32)(totalSamplesSize>>20), (COVER_MAX_SAMPLES_SIZE >> 20));
     return 0;
   }
+  /* Check if there are at least 5 training samples */
+  if (nbTrainSamples < 5) {
+    DISPLAYLEVEL(1, "Total number of training samples is %u and is invalid.", nbTrainSamples);
+    return 0;
+  }
+  /* Check if there's testing sample */
+  if (nbTestSamples < 1) {
+    DISPLAYLEVEL(1, "Total number of testing samples is %u and is invalid.", nbTestSamples);
+    return 0;
+  }
   /* Zero the context */
   memset(ctx, 0, sizeof(*ctx));
-  DISPLAYLEVEL(2, "Training on %u samples of total size %u\n", nbSamples,
+  DISPLAYLEVEL(2, "Training on %u samples of total size %u\n", nbTrainSamples,
-               (U32)totalSamplesSize);
+               (U32)trainingSamplesSize);
+  DISPLAYLEVEL(2, "Testing on %u samples of total size %u\n", nbTestSamples,
+               (U32)testSamplesSize);
   ctx->samples = samples;
   ctx->samplesSizes = samplesSizes;
   ctx->nbSamples = nbSamples;
+  ctx->nbTrainSamples = nbTrainSamples;
+  ctx->nbTestSamples = nbTestSamples;
   /* Partial suffix array */
-  ctx->suffixSize = totalSamplesSize - MAX(d, sizeof(U64)) + 1;
+  ctx->suffixSize = trainingSamplesSize - MAX(d, sizeof(U64)) + 1;
   ctx->suffix = (U32 *)malloc(ctx->suffixSize * sizeof(U32));
   /* Maps index to the dmerID */
   ctx->dmerAt = (U32 *)malloc(ctx->suffixSize * sizeof(U32));
@@ -563,7 +589,7 @@ static int COVER_ctx_init(COVER_ctx_t *ctx, const void *samplesBuffer,
   ctx->freqs = NULL;
   ctx->d = d;
 
-  /* Fill offsets from the samlesSizes */
+  /* Fill offsets from the samplesSizes */
   {
     U32 i;
     ctx->offsets[0] = 0;
@@ -665,7 +691,7 @@ ZDICTLIB_API size_t ZDICT_trainFromBuffer_cover(
   BYTE* const dict = (BYTE*)dictBuffer;
   COVER_ctx_t ctx;
   COVER_map_t activeDmers;
-
+  parameters.splitPoint = 1.0;
   /* Initialize global data */
   g_displayLevel = parameters.zParams.notificationLevel;
   /* Checks */
@@ -684,7 +710,7 @@ ZDICTLIB_API size_t ZDICT_trainFromBuffer_cover(
   }
   /* Initialize context and activeDmers */
   if (!COVER_ctx_init(&ctx, samplesBuffer, samplesSizes, nbSamples,
-                      parameters.d)) {
+                      parameters.d, parameters.splitPoint)) {
     return ERROR(GENERIC);
   }
   if (!COVER_map_init(&activeDmers, parameters.k - parameters.d + 1)) {
@@ -839,7 +865,7 @@ typedef struct COVER_tryParameters_data_s {
 } COVER_tryParameters_data_t;
 
 /**
- * Tries a set of parameters and upates the COVER_best_t with the results.
+ * Tries a set of parameters and updates the COVER_best_t with the results.
  * This function is thread safe if zstd is compiled with multithreaded support.
  * It takes its parameters as an *OWNING* opaque pointer to support threading.
  */
@@ -870,7 +896,7 @@ static void COVER_tryParameters(void *opaque) {
                                               dictBufferCapacity, parameters);
     dictBufferCapacity = ZDICT_finalizeDictionary(
         dict, dictBufferCapacity, dict + tail, dictBufferCapacity - tail,
-        ctx->samples, ctx->samplesSizes, (unsigned)ctx->nbSamples,
+        ctx->samples, ctx->samplesSizes, (unsigned)ctx->nbTrainSamples,
         parameters.zParams);
     if (ZDICT_isError(dictBufferCapacity)) {
       DISPLAYLEVEL(1, "Failed to finalize dictionary\n");
@@ -889,7 +915,8 @@ static void COVER_tryParameters(void *opaque) {
     /* Allocate dst with enough space to compress the maximum sized sample */
     {
       size_t maxSampleSize = 0;
-      for (i = 0; i < ctx->nbSamples; ++i) {
+      i = parameters.splitPoint < 1.0 ? ctx->nbTrainSamples : 0;
+      for (; i < ctx->nbSamples; ++i) {
         maxSampleSize = MAX(ctx->samplesSizes[i], maxSampleSize);
       }
       dstCapacity = ZSTD_compressBound(maxSampleSize);
@@ -904,7 +931,8 @@ static void COVER_tryParameters(void *opaque) {
     }
     /* Compress each sample and sum their sizes (or error) */
     totalCompressedSize = dictBufferCapacity;
-    for (i = 0; i < ctx->nbSamples; ++i) {
+    i = parameters.splitPoint < 1.0 ? ctx->nbTrainSamples : 0;
+    for (; i < ctx->nbSamples; ++i) {
       const size_t size = ZSTD_compress_usingCDict(
           cctx, dst, dstCapacity, ctx->samples + ctx->offsets[i],
           ctx->samplesSizes[i], cdict);
@@ -941,6 +969,8 @@ ZDICTLIB_API size_t ZDICT_optimizeTrainFromBuffer_cover(
     ZDICT_cover_params_t *parameters) {
   /* constants */
   const unsigned nbThreads = parameters->nbThreads;
+  const double splitPoint =
+      parameters->splitPoint <= 0.0 ? DEFAULT_SPLITPOINT : parameters->splitPoint;
   const unsigned kMinD = parameters->d == 0 ? 6 : parameters->d;
   const unsigned kMaxD = parameters->d == 0 ? 8 : parameters->d;
   const unsigned kMinK = parameters->k == 0 ? 50 : parameters->k;
@@ -958,6 +988,10 @@ ZDICTLIB_API size_t ZDICT_optimizeTrainFromBuffer_cover(
   POOL_ctx *pool = NULL;
 
   /* Checks */
+  if (splitPoint <= 0 || splitPoint > 1) {
+    LOCALDISPLAYLEVEL(displayLevel, 1, "Incorrect parameters\n");
+    return ERROR(GENERIC);
+  }
   if (kMinK < kMaxD || kMaxK < kMinK) {
     LOCALDISPLAYLEVEL(displayLevel, 1, "Incorrect parameters\n");
     return ERROR(GENERIC);
@@ -988,7 +1022,7 @@ ZDICTLIB_API size_t ZDICT_optimizeTrainFromBuffer_cover(
     /* Initialize the context for this value of d */
     COVER_ctx_t ctx;
     LOCALDISPLAYLEVEL(displayLevel, 3, "d=%u\n", d);
-    if (!COVER_ctx_init(&ctx, samplesBuffer, samplesSizes, nbSamples, d)) {
+    if (!COVER_ctx_init(&ctx, samplesBuffer, samplesSizes, nbSamples, d, splitPoint)) {
       LOCALDISPLAYLEVEL(displayLevel, 1, "Failed to initialize context\n");
       COVER_best_destroy(&best);
       POOL_free(pool);
@@ -1013,6 +1047,7 @@ ZDICTLIB_API size_t ZDICT_optimizeTrainFromBuffer_cover(
       data->parameters = *parameters;
       data->parameters.k = k;
       data->parameters.d = d;
+      data->parameters.splitPoint = splitPoint;
       data->parameters.steps = kSteps;
       data->parameters.zParams.notificationLevel = g_displayLevel;
       /* Check the parameters */

lib/dictBuilder/zdict.h

@@ -86,6 +86,7 @@ typedef struct {
     unsigned d;                  /* dmer size : constraint: 0 < d <= k : Reasonable range [6, 16] */
     unsigned steps;              /* Number of steps : Only used for optimization : 0 means default (32) : Higher means more parameters checked */
     unsigned nbThreads;          /* Number of threads : constraint: 0 < nbThreads : 1 means single-threaded : Only used for optimization : Ignored if ZSTD_MULTITHREAD is not defined */
+    double splitPoint;           /* Percentage of samples used for training: the first nbSamples * splitPoint samples will be used to training, the last nbSamples * (1 - splitPoint) samples will be used for testing, 0 means default (1.0), 1.0 when all samples are used for both training and testing */
     ZDICT_params_t zParams;
 } ZDICT_cover_params_t;
 

programs/README.md

@@ -150,7 +150,7 @@ Advanced arguments :
 
 Dictionary builder :
 --train ## : create a dictionary from a training set of files
---train-cover[=k=#,d=#,steps=#] : use the cover algorithm with optional args
+--train-cover[=k=#,d=#,steps=#,split=#] : use the cover algorithm with optional args
 --train-legacy[=s=#] : use the legacy algorithm with selectivity (default: 9)
  -o file : `file` is dictionary name (default: dictionary)
 --maxdict=# : limit dictionary to specified size (default: 112640)

programs/bench.c

@@ -42,6 +42,7 @@
 #include "datagen.h"     /* RDG_genBuffer */
 #include "xxhash.h"
 #include "bench.h"
+#include "zstd_errors.h"
 
 
 /* *************************************
@@ -64,9 +65,6 @@
 
 static const size_t maxMemory = (sizeof(size_t)==4)  ?  (2 GB - 64 MB) : (size_t)(1ULL << ((sizeof(size_t)*8)-31));
 
-static U32 g_compressibilityDefault = 50;
-
-
 /* *************************************
 *  console display
 ***************************************/
@@ -90,88 +88,60 @@ static UTIL_time_t g_displayClock = UTIL_TIME_INITIALIZER;
 #  define DEBUG 0
 #endif
 #define DEBUGOUTPUT(...) { if (DEBUG) DISPLAY(__VA_ARGS__); }
-#define EXM_THROW(error, ...)  {                      \
+
+#define EXM_THROW_INT(errorNum, ...)  {               \
     DEBUGOUTPUT("%s: %i: \n", __FILE__, __LINE__);    \
-    DISPLAYLEVEL(1, "Error %i : ", error);            \
+    DISPLAYLEVEL(1, "Error %i : ", errorNum);         \
     DISPLAYLEVEL(1, __VA_ARGS__);                     \
     DISPLAYLEVEL(1, " \n");                           \
-    exit(error);                                      \
+    return errorNum;                                  \
 }
 
+#define EXM_THROW(errorNum, retType, ...)  {          \
+    retType r;                                        \
+    memset(&r, 0, sizeof(retType));                   \
+    DEBUGOUTPUT("%s: %i: \n", __FILE__, __LINE__);    \
+    DISPLAYLEVEL(1, "Error %i : ", errorNum);         \
+    DISPLAYLEVEL(1, __VA_ARGS__);                     \
+    DISPLAYLEVEL(1, " \n");                           \
+    r.error = errorNum;                               \
+    return r;                                         \
+}
+
+/* error without displaying */
+#define EXM_THROW_ND(errorNum, retType, ...)  {       \
+    retType r;                                        \
+    memset(&r, 0, sizeof(retType));                   \
+    DEBUGOUTPUT("%s: %i: \n", __FILE__, __LINE__);    \
+    DEBUGOUTPUT("Error %i : ", errorNum);             \
+    DEBUGOUTPUT(__VA_ARGS__);                         \
+    DEBUGOUTPUT(" \n");                               \
+    r.error = errorNum;                               \
+    return r;                                         \
+}
 
 /* *************************************
 *  Benchmark Parameters
 ***************************************/
-static int g_additionalParam = 0;
-static U32 g_decodeOnly = 0;
 
-void BMK_setAdditionalParam(int additionalParam) { g_additionalParam=additionalParam; }
+BMK_advancedParams_t BMK_initAdvancedParams(void) { 
-
+    BMK_advancedParams_t res = { 
-
+        BMK_both, /* mode */
-//TODO : Deal with DISPLAYLEVEL for all these set functions
+        BMK_timeMode, /* loopMode */
-
+        BMK_TIMETEST_DEFAULT_S, /* nbSeconds */
-static U32 g_nbSeconds = BMK_TIMETEST_DEFAULT_S;
+        0, /* blockSize */
-
+        0, /* nbWorkers */
-void BMK_setNbSeconds(unsigned nbSeconds)
+        0, /* realTime */
-{
+        0, /* additionalParam */
-    g_nbSeconds = nbSeconds;
+        0, /* ldmFlag */ 
-    DISPLAY("- test >= %u seconds per compression / decompression - \n", g_nbSeconds);
+        0, /* ldmMinMatch */
+        0, /* ldmHashLog */
+        0, /* ldmBuckSizeLog */
+        0  /* ldmHashEveryLog */
+    };
+    return res;
 }
 
-static size_t g_blockSize = 0;
-
-void BMK_setBlockSize(size_t blockSize)
-{
-    g_blockSize = blockSize;
-    if (g_blockSize) DISPLAY("using blocks of size %u KB \n", (U32)(blockSize>>10));
-}
-
-void BMK_setDecodeOnlyMode(unsigned decodeFlag) { g_decodeOnly = (decodeFlag>0); }
-
-static U32 g_nbWorkers = 0;
-
-void BMK_setNbWorkers(unsigned nbWorkers) {
-#ifndef ZSTD_MULTITHREAD
-    if (nbWorkers > 0) DISPLAY("Note : multi-threading is disabled \n");
-#endif
-    g_nbWorkers = nbWorkers;
-}
-
-static U32 g_realTime = 0;
-void BMK_setRealTime(unsigned priority) {
-    g_realTime = (priority>0);
-}
-
-static U32 g_separateFiles = 0;
-void BMK_setSeparateFiles(unsigned separate) {
-    g_separateFiles = (separate>0);
-}
-
-static U32 g_ldmFlag = 0;
-void BMK_setLdmFlag(unsigned ldmFlag) {
-    g_ldmFlag = ldmFlag;
-}
-
-static U32 g_ldmMinMatch = 0;
-void BMK_setLdmMinMatch(unsigned ldmMinMatch) {
-    g_ldmMinMatch = ldmMinMatch;
-}
-
-static U32 g_ldmHashLog = 0;
-void BMK_setLdmHashLog(unsigned ldmHashLog) {
-    g_ldmHashLog = ldmHashLog;
-}
-
-#define BMK_LDM_PARAM_NOTSET 9999
-static U32 g_ldmBucketSizeLog = BMK_LDM_PARAM_NOTSET;
-void BMK_setLdmBucketSizeLog(unsigned ldmBucketSizeLog) {
-    g_ldmBucketSizeLog = ldmBucketSizeLog;
-}
-
-static U32 g_ldmHashEveryLog = BMK_LDM_PARAM_NOTSET;
-void BMK_setLdmHashEveryLog(unsigned ldmHashEveryLog) {
-    g_ldmHashEveryLog = ldmHashEveryLog;
-}
 
 /* ********************************************************
 *  Bench functions
@@ -186,140 +156,32 @@ typedef struct {
     size_t resSize;
 } blockParam_t;
 
+struct  BMK_timeState_t{
+    unsigned nbLoops;
+    U64 timeRemaining;
+    UTIL_time_t coolTime;
+    U64 fastestTime;
+};
+
 #undef MIN
 #undef MAX
 #define MIN(a,b)    ((a) < (b) ? (a) : (b))
 #define MAX(a,b)    ((a) > (b) ? (a) : (b))
 
-BMK_return_t BMK_benchMem(const void* srcBuffer, size_t srcSize,
+static void BMK_initCCtx(ZSTD_CCtx* ctx, 
-                        const size_t* fileSizes, unsigned nbFiles,
+    const void* dictBuffer, size_t dictBufferSize, int cLevel, 
-                        const int cLevel, const ZSTD_compressionParameters* comprParams,
+    const ZSTD_compressionParameters* comprParams, const BMK_advancedParams_t* adv) {
-                        const void* dictBuffer, size_t dictBufferSize,
+    if (adv->nbWorkers==1) {
-                        ZSTD_CCtx* ctx, ZSTD_DCtx* dctx,
+        ZSTD_CCtx_setParameter(ctx, ZSTD_p_nbWorkers, 0);
-                        int displayLevel, const char* displayName)
-
-{
-    size_t const blockSize = ((g_blockSize>=32 && !g_decodeOnly) ? g_blockSize : srcSize) + (!srcSize) /* avoid div by 0 */ ;
-    U32 const maxNbBlocks = (U32) ((srcSize + (blockSize-1)) / blockSize) + nbFiles;
-    blockParam_t* const blockTable = (blockParam_t*) malloc(maxNbBlocks * sizeof(blockParam_t));
-    size_t const maxCompressedSize = ZSTD_compressBound(srcSize) + (maxNbBlocks * 1024);   /* add some room for safety */
-    void* const compressedBuffer = malloc(maxCompressedSize);
-    void* resultBuffer = malloc(srcSize);
-    BMK_return_t results;
-
-    size_t const loadedCompressedSize = srcSize;
-    size_t cSize = 0;
-    double ratio = 0.;
-    U32 nbBlocks;
-
-    /* checks */
-    if (!compressedBuffer || !resultBuffer || !blockTable)
-        EXM_THROW(31, "allocation error : not enough memory");
-
-    if(!ctx || !dctx) 
-        EXM_THROW(31, "error: passed in null context");
-
-    /* init */
-    if (strlen(displayName)>17) displayName += strlen(displayName)-17;   /* display last 17 characters */
-    if (g_nbWorkers==1) g_nbWorkers=0;   /* prefer synchronous mode */
-
-    if (g_decodeOnly) {  /* benchmark only decompression : source must be already compressed */
-        const char* srcPtr = (const char*)srcBuffer;
-        U64 totalDSize64 = 0;
-        U32 fileNb;
-        for (fileNb=0; fileNb<nbFiles; fileNb++) {
-            U64 const fSize64 = ZSTD_findDecompressedSize(srcPtr, fileSizes[fileNb]);
-            if (fSize64==0) EXM_THROW(32, "Impossible to determine original size ");
-            totalDSize64 += fSize64;
-            srcPtr += fileSizes[fileNb];
-        }
-        {   size_t const decodedSize = (size_t)totalDSize64;
-            if (totalDSize64 > decodedSize) EXM_THROW(32, "original size is too large");   /* size_t overflow */
-            free(resultBuffer);
-            resultBuffer = malloc(decodedSize);
-            if (!resultBuffer) EXM_THROW(33, "not enough memory");
-            cSize = srcSize;
-            srcSize = decodedSize;
-            ratio = (double)srcSize / (double)cSize;
-    }   }
-
-    /* Init blockTable data */
-    {   const char* srcPtr = (const char*)srcBuffer;
-        char* cPtr = (char*)compressedBuffer;
-        char* resPtr = (char*)resultBuffer;
-        U32 fileNb;
-        for (nbBlocks=0, fileNb=0; fileNb<nbFiles; fileNb++) {
-            size_t remaining = fileSizes[fileNb];
-            U32 const nbBlocksforThisFile = g_decodeOnly ? 1 : (U32)((remaining + (blockSize-1)) / blockSize);
-            U32 const blockEnd = nbBlocks + nbBlocksforThisFile;
-            for ( ; nbBlocks<blockEnd; nbBlocks++) {
-                size_t const thisBlockSize = MIN(remaining, blockSize);
-                blockTable[nbBlocks].srcPtr = (const void*)srcPtr;
-                blockTable[nbBlocks].srcSize = thisBlockSize;
-                blockTable[nbBlocks].cPtr = (void*)cPtr;
-                blockTable[nbBlocks].cRoom = g_decodeOnly ? thisBlockSize : ZSTD_compressBound(thisBlockSize);
-                blockTable[nbBlocks].cSize = blockTable[nbBlocks].cRoom;
-                blockTable[nbBlocks].resPtr = (void*)resPtr;
-                blockTable[nbBlocks].resSize = g_decodeOnly ? (size_t) ZSTD_findDecompressedSize(srcPtr, thisBlockSize) : thisBlockSize;
-                srcPtr += thisBlockSize;
-                cPtr += blockTable[nbBlocks].cRoom;
-                resPtr += thisBlockSize;
-                remaining -= thisBlockSize;
-    }   }   }
-
-    /* warmimg up memory */
-    if (g_decodeOnly) {
-        memcpy(compressedBuffer, srcBuffer, loadedCompressedSize);
     } else {
-        RDG_genBuffer(compressedBuffer, maxCompressedSize, 0.10, 0.50, 1);
+        ZSTD_CCtx_setParameter(ctx, ZSTD_p_nbWorkers, adv->nbWorkers);
     }
-
-    /* Bench */
-    {   U64 fastestC = (U64)(-1LL), fastestD = (U64)(-1LL);
-        U64 const crcOrig = g_decodeOnly ? 0 : XXH64(srcBuffer, srcSize, 0);
-        UTIL_time_t coolTime;
-        U64 const maxTime = (g_nbSeconds * TIMELOOP_NANOSEC) + 1;
-        U32 nbDecodeLoops = (U32)((100 MB) / (srcSize+1)) + 1;  /* initial conservative speed estimate */
-        U32 nbCompressionLoops = (U32)((2 MB) / (srcSize+1)) + 1;  /* initial conservative speed estimate */
-        U64 totalCTime=0, totalDTime=0;
-        U32 cCompleted=g_decodeOnly, dCompleted=0;
-#       define NB_MARKS 4
-        const char* const marks[NB_MARKS] = { " |", " /", " =",  "\\" };
-        U32 markNb = 0;
-
-        coolTime = UTIL_getTime();
-        DISPLAYLEVEL(2, "\r%79s\r", "");
-        while (!cCompleted || !dCompleted) {
-
-            /* overheat protection */
-            if (UTIL_clockSpanMicro(coolTime) > ACTIVEPERIOD_MICROSEC) {
-                DISPLAYLEVEL(2, "\rcooling down ...    \r");
-                UTIL_sleep(COOLPERIOD_SEC);
-                coolTime = UTIL_getTime();
-            }
-
-            if (!g_decodeOnly) {
-                /* Compression */
-                DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->\r", marks[markNb], displayName, (U32)srcSize);
-                if (!cCompleted) memset(compressedBuffer, 0xE5, maxCompressedSize);  /* warm up and erase result buffer */
-
-                UTIL_sleepMilli(5);  /* give processor time to other processes */
-                UTIL_waitForNextTick();
-
-                if (!cCompleted) {   /* still some time to do compression tests */
-                    U32 nbLoops = 0;
-                    UTIL_time_t const clockStart = UTIL_getTime();
-                    ZSTD_CCtx_setParameter(ctx, ZSTD_p_nbWorkers, g_nbWorkers);
     ZSTD_CCtx_setParameter(ctx, ZSTD_p_compressionLevel, cLevel);
-                    ZSTD_CCtx_setParameter(ctx, ZSTD_p_enableLongDistanceMatching, g_ldmFlag);
+    ZSTD_CCtx_setParameter(ctx, ZSTD_p_enableLongDistanceMatching, adv->ldmFlag);
-                    ZSTD_CCtx_setParameter(ctx, ZSTD_p_ldmMinMatch, g_ldmMinMatch);
+    ZSTD_CCtx_setParameter(ctx, ZSTD_p_ldmMinMatch, adv->ldmMinMatch);
-                    ZSTD_CCtx_setParameter(ctx, ZSTD_p_ldmHashLog, g_ldmHashLog);
+    ZSTD_CCtx_setParameter(ctx, ZSTD_p_ldmHashLog, adv->ldmHashLog);
-                    if (g_ldmBucketSizeLog != BMK_LDM_PARAM_NOTSET) {
+    ZSTD_CCtx_setParameter(ctx, ZSTD_p_ldmBucketSizeLog, adv->ldmBucketSizeLog);
-                      ZSTD_CCtx_setParameter(ctx, ZSTD_p_ldmBucketSizeLog, g_ldmBucketSizeLog);
+    ZSTD_CCtx_setParameter(ctx, ZSTD_p_ldmHashEveryLog, adv->ldmHashEveryLog);
-                    }
-                    if (g_ldmHashEveryLog != BMK_LDM_PARAM_NOTSET) {
-                      ZSTD_CCtx_setParameter(ctx, ZSTD_p_ldmHashEveryLog, g_ldmHashEveryLog);
-                    }
     ZSTD_CCtx_setParameter(ctx, ZSTD_p_windowLog, comprParams->windowLog);
     ZSTD_CCtx_setParameter(ctx, ZSTD_p_hashLog, comprParams->hashLog);
     ZSTD_CCtx_setParameter(ctx, ZSTD_p_chainLog, comprParams->chainLog);
@@ -328,127 +190,460 @@ BMK_return_t BMK_benchMem(const void* srcBuffer, size_t srcSize,
     ZSTD_CCtx_setParameter(ctx, ZSTD_p_targetLength, comprParams->targetLength);
     ZSTD_CCtx_setParameter(ctx, ZSTD_p_compressionStrategy, comprParams->strategy);
     ZSTD_CCtx_loadDictionary(ctx, dictBuffer, dictBufferSize);
+}
 
-                    if (!g_nbSeconds) nbCompressionLoops=1;
+
-                    for (nbLoops=0; nbLoops<nbCompressionLoops; nbLoops++) {
+static void BMK_initDCtx(ZSTD_DCtx* dctx, 
-                        U32 blockNb;
+    const void* dictBuffer, size_t dictBufferSize) {
-                        for (blockNb=0; blockNb<nbBlocks; blockNb++) {
+    ZSTD_DCtx_loadDictionary(dctx, dictBuffer, dictBufferSize);
-#if 0   /* direct compression function, for occasional comparison */
+}
-                            ZSTD_parameters const params = ZSTD_getParams(cLevel, blockTable[blockNb].srcSize, dictBufferSize);
+
-                            blockTable[blockNb].cSize = ZSTD_compress_advanced(ctx,
+typedef struct {
-                                                            blockTable[blockNb].cPtr, blockTable[blockNb].cRoom,
+    ZSTD_CCtx* ctx;
-                                                            blockTable[blockNb].srcPtr, blockTable[blockNb].srcSize,
+    const void* dictBuffer;
-                                                            dictBuffer, dictBufferSize,
+    size_t dictBufferSize;
-                                                            params);
+    int cLevel;
-#else
+    const ZSTD_compressionParameters* comprParams;
+    const BMK_advancedParams_t* adv;
+} BMK_initCCtxArgs;
+
+static size_t local_initCCtx(void* payload) {
+    BMK_initCCtxArgs* ag = (BMK_initCCtxArgs*)payload;
+    BMK_initCCtx(ag->ctx, ag->dictBuffer, ag->dictBufferSize, ag->cLevel, ag->comprParams, ag->adv);
+    return 0;
+}
+
+typedef struct {
+    ZSTD_DCtx* dctx;
+    const void* dictBuffer;
+    size_t dictBufferSize;
+} BMK_initDCtxArgs;
+
+static size_t local_initDCtx(void* payload) {
+    BMK_initDCtxArgs* ag = (BMK_initDCtxArgs*)payload;
+    BMK_initDCtx(ag->dctx, ag->dictBuffer, ag->dictBufferSize);
+    return 0;
+}
+
+/* additional argument is just the context */
+static size_t local_defaultCompress(
+    const void* srcBuffer, size_t srcSize, 
+    void* dstBuffer, size_t dstSize, 
+    void* addArgs) {
     size_t moreToFlush = 1;
-                            ZSTD_outBuffer out;
+    ZSTD_CCtx* ctx = (ZSTD_CCtx*)addArgs;
     ZSTD_inBuffer in;
-                            in.src = blockTable[blockNb].srcPtr;
+    ZSTD_outBuffer out;
-                            in.size = blockTable[blockNb].srcSize;
+    in.src = srcBuffer;
+    in.size = srcSize;
     in.pos = 0;
-                            out.dst = blockTable[blockNb].cPtr;
+    out.dst = dstBuffer;
-                            out.size = blockTable[blockNb].cRoom;
+    out.size = dstSize;
     out.pos = 0;
     while (moreToFlush) {
-                                moreToFlush = ZSTD_compress_generic(ctx,
+        if(out.pos == out.size) {
-                                                    &out, &in, ZSTD_e_end);
+            return (size_t)-ZSTD_error_dstSize_tooSmall;
-                                if (ZSTD_isError(moreToFlush))
-                                    EXM_THROW(1, "ZSTD_compress_generic() error : %s",
-                                                ZSTD_getErrorName(moreToFlush));
         }
-                            blockTable[blockNb].cSize = out.pos;
+        moreToFlush = ZSTD_compress_generic(ctx, &out, &in, ZSTD_e_end);
-#endif
+        if (ZSTD_isError(moreToFlush)) {
-                    }   }
+            return moreToFlush;
-                    {   U64 const loopDuration = UTIL_clockSpanNano(clockStart);
+        }
-                        if (loopDuration > 0) {
+    }
-                            if (loopDuration < fastestC * nbCompressionLoops)
+    return out.pos;
-                                fastestC = loopDuration / nbCompressionLoops;
-                            nbCompressionLoops = (U32)(TIMELOOP_NANOSEC / fastestC) + 1;
-                        } else {
-                            assert(nbCompressionLoops < 40000000);  /* avoid overflow */
-                            nbCompressionLoops *= 100;
 }
-                        totalCTime += loopDuration;
-                        cCompleted = (totalCTime >= maxTime);  /* end compression tests */
-                }   }
 
-                cSize = 0;
+/* additional argument is just the context */
-                { U32 blockNb; for (blockNb=0; blockNb<nbBlocks; blockNb++) cSize += blockTable[blockNb].cSize; }
+static size_t local_defaultDecompress(
-                ratio = (double)srcSize / (double)cSize;
+    const void* srcBuffer, size_t srcSize, 
-                results.result.cSize = cSize;
+    void* dstBuffer, size_t dstSize, 
-                markNb = (markNb+1) % NB_MARKS;
+    void* addArgs) {
-                {   int const ratioAccuracy = (ratio < 10.) ? 3 : 2;
+    size_t moreToFlush = 1;
-                    double const compressionSpeed = ((double)srcSize / fastestC) * 1000;
+    ZSTD_DCtx* dctx = (ZSTD_DCtx*)addArgs;
-                    int const cSpeedAccuracy = (compressionSpeed < 10.) ? 2 : 1;
+    ZSTD_inBuffer in;
-                    results.result.cSpeed = compressionSpeed * 1000000;
+    ZSTD_outBuffer out;
-                    DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.*f),%6.*f MB/s\r",
+    in.src = srcBuffer;
-                            marks[markNb], displayName, (U32)srcSize, (U32)cSize,
+    in.size = srcSize;
-                            ratioAccuracy, ratio,
+    in.pos = 0;
-                            cSpeedAccuracy, compressionSpeed );
+    out.dst = dstBuffer;
+    out.size = dstSize;
+    out.pos = 0;
+    while (moreToFlush) {
+        if(out.pos == out.size) {
+            return (size_t)-ZSTD_error_dstSize_tooSmall;
         }
-            }  /* if (!g_decodeOnly) */
+        moreToFlush = ZSTD_decompress_generic(dctx,
+                            &out, &in);
+        if (ZSTD_isError(moreToFlush)) {
+            return moreToFlush;
+        }
+    }
+    return out.pos;
 
-#if 0       /* disable decompression test */
+}
-            dCompleted=1;
+
-            (void)totalDTime; (void)fastestD; (void)crcOrig;   /* unused when decompression disabled */
+/* initFn will be measured once, bench fn will be measured x times */
-#else
+/* benchFn should return error value or out Size */
-            /* Decompression */
+/* takes # of blocks and list of size & stuff for each. */
-            if (!dCompleted) memset(resultBuffer, 0xD6, srcSize);  /* warm result buffer */
+/* only does looping */
+/* note time per loop could be zero if interval too short */
+BMK_customReturn_t BMK_benchFunction(
+    BMK_benchFn_t benchFn, void* benchPayload,
+    BMK_initFn_t initFn, void* initPayload,
+    size_t blockCount,
+    const void* const * const srcBlockBuffers, const size_t* srcBlockSizes,
+    void* const * const dstBlockBuffers, const size_t* dstBlockCapacities,
+    unsigned nbLoops) {
+    size_t srcSize = 0, dstSize = 0, ind = 0;
+    U64 totalTime;
+
+    BMK_customReturn_t retval;
+    UTIL_time_t clockStart;
+
+    {
+        unsigned i;
+        for(i = 0; i < blockCount; i++) {
+            memset(dstBlockBuffers[i], 0xE5, dstBlockCapacities[i]);  /* warm up and erase result buffer */
+        }
 
         UTIL_sleepMilli(5);  /* give processor time to other processes */
         UTIL_waitForNextTick();
-
-            if (!dCompleted) {
-                U32 nbLoops = 0;
-                ZSTD_DDict* const ddict = ZSTD_createDDict(dictBuffer, dictBufferSize);
-                UTIL_time_t const clockStart = UTIL_getTime();
-                if (!ddict) EXM_THROW(2, "ZSTD_createDDict() allocation failure");
-                if (!g_nbSeconds) nbDecodeLoops = 1;
-                for (nbLoops=0; nbLoops < nbDecodeLoops; nbLoops++) {
-                    U32 blockNb;
-                    for (blockNb=0; blockNb<nbBlocks; blockNb++) {
-                        size_t const regenSize = ZSTD_decompress_usingDDict(dctx,
-                            blockTable[blockNb].resPtr, blockTable[blockNb].resSize,
-                            blockTable[blockNb].cPtr, blockTable[blockNb].cSize,
-                            ddict);
-                        if (ZSTD_isError(regenSize)) {
-                            EXM_THROW(2, "ZSTD_decompress_usingDDict() failed on block %u of size %u : %s  \n",
-                                      blockNb, (U32)blockTable[blockNb].cSize, ZSTD_getErrorName(regenSize));
     }
-                        blockTable[blockNb].resSize = regenSize;
+
-                }   }
+    if(!nbLoops) {
-                ZSTD_freeDDict(ddict);
+        EXM_THROW_ND(1, BMK_customReturn_t, "nbLoops must be nonzero \n");
-                {   U64 const loopDuration = UTIL_clockSpanNano(clockStart);
+    }
+
+    for(ind = 0; ind < blockCount; ind++) {
+        srcSize += srcBlockSizes[ind];
+    }
+
+    {      
+        unsigned i, j, firstIter = 1;
+        clockStart = UTIL_getTime();
+        if(initFn != NULL) { initFn(initPayload); }
+        for(i = 0; i < nbLoops; i++) {
+            for(j = 0; j < blockCount; j++) {
+                size_t res = benchFn(srcBlockBuffers[j], srcBlockSizes[j], dstBlockBuffers[j], dstBlockCapacities[j], benchPayload);
+                if(ZSTD_isError(res)) {
+                    EXM_THROW_ND(2, BMK_customReturn_t, "Function benchmarking failed on block %u of size %u : %s  \n",
+                        j, (U32)dstBlockCapacities[j], ZSTD_getErrorName(res));
+                } else if(firstIter) {
+                    dstSize += res;
+                }
+            }
+            firstIter = 0;
+        }
+        totalTime = UTIL_clockSpanNano(clockStart);
+    }
+
+    retval.error = 0;
+    retval.result.nanoSecPerRun = totalTime / nbLoops;
+    retval.result.sumOfReturn = dstSize;
+    return retval;
+} 
+
+#define MINUSABLETIME 500000000ULL /* 0.5 seconds in ns */
+
+void BMK_resetTimeState(BMK_timedFnState_t* r, unsigned nbSeconds) {
+    r->nbLoops = 1;
+    r->timeRemaining = (U64)nbSeconds * TIMELOOP_NANOSEC;
+    r->coolTime = UTIL_getTime();
+    r->fastestTime = (U64)(-1LL);
+}
+
+BMK_timedFnState_t* BMK_createTimeState(unsigned nbSeconds) {
+    BMK_timedFnState_t* r = (BMK_timedFnState_t*)malloc(sizeof(struct BMK_timeState_t));
+    BMK_resetTimeState(r, nbSeconds);
+    return r;
+}
+
+void BMK_freeTimeState(BMK_timedFnState_t* state) {
+    free(state);
+}
+
+BMK_customTimedReturn_t BMK_benchFunctionTimed(
+    BMK_timedFnState_t* cont,
+    BMK_benchFn_t benchFn, void* benchPayload,
+    BMK_initFn_t initFn, void* initPayload,
+    size_t blockCount,
+    const void* const* const srcBlockBuffers, const size_t* srcBlockSizes,
+    void* const* const dstBlockBuffers, const size_t* dstBlockCapacities) 
+{
+    U64 fastest = cont->fastestTime;
+    int completed = 0;
+    BMK_customTimedReturn_t r;
+    r.completed = 0;
+
+    while(!r.completed && !completed)
+    {
+        /* Overheat protection */
+        if (UTIL_clockSpanMicro(cont->coolTime) > ACTIVEPERIOD_MICROSEC) {
+            DEBUGOUTPUT("\rcooling down ...    \r");
+            UTIL_sleep(COOLPERIOD_SEC);
+            cont->coolTime = UTIL_getTime();
+        }
+
+        r.result = BMK_benchFunction(benchFn, benchPayload, initFn, initPayload,
+        blockCount, srcBlockBuffers, srcBlockSizes, dstBlockBuffers, dstBlockCapacities, cont->nbLoops);
+        if(r.result.error) { /* completed w/ error */
+            r.completed = 1;
+            return r;
+        }
+
+        {   U64 const loopDuration = r.result.result.nanoSecPerRun * cont->nbLoops;
+            r.completed = (cont->timeRemaining <= loopDuration);
+            cont->timeRemaining -= loopDuration;
             if (loopDuration > 0) { 
-                        if (loopDuration < fastestD * nbDecodeLoops)
+                fastest = MIN(fastest, r.result.result.nanoSecPerRun);
-                            fastestD = loopDuration / nbDecodeLoops;
+                if(loopDuration >= MINUSABLETIME) {
-                        nbDecodeLoops = (U32)(TIMELOOP_NANOSEC / fastestD) + 1;
+                    r.result.result.nanoSecPerRun = fastest;
+                    cont->fastestTime = fastest;
+                }
+                cont->nbLoops = (U32)(TIMELOOP_NANOSEC / r.result.result.nanoSecPerRun) + 1;
             } else {
-                        assert(nbDecodeLoops < 40000000);  /* avoid overflow */
+                const unsigned multiplier = 2;
-                        nbDecodeLoops *= 100;
+                assert(cont->nbLoops < ((unsigned)-1) / multiplier);  /* avoid overflow */
+                cont->nbLoops *= multiplier;
+            }
+            if(loopDuration < MINUSABLETIME) { /* don't report results which have time too low */
+                continue;
             }
-                    totalDTime += loopDuration;
-                    dCompleted = (totalDTime >= maxTime);
-            }   }
 
-            markNb = (markNb+1) % NB_MARKS;
+        }
-            {   int const ratioAccuracy = (ratio < 10.) ? 3 : 2;
+        completed = 1;
-                double const compressionSpeed = ((double)srcSize / fastestC) * 1000;
+    }
+    return r;
+}
+
+/* benchMem with no allocation */
+static BMK_return_t BMK_benchMemAdvancedNoAlloc(
+    const void ** const srcPtrs, size_t* const srcSizes,
+    void** const cPtrs, size_t* const cSizes,
+    void** const resPtrs, size_t* const resSizes,
+    void* resultBuffer, void* compressedBuffer,
+    const size_t maxCompressedSize,
+    BMK_timedFnState_t* timeStateCompress, BMK_timedFnState_t* timeStateDecompress,
+
+    const void* srcBuffer, size_t srcSize,
+    const size_t* fileSizes, unsigned nbFiles,
+    const int cLevel, const ZSTD_compressionParameters* comprParams,
+    const void* dictBuffer, size_t dictBufferSize,
+    ZSTD_CCtx* ctx, ZSTD_DCtx* dctx,
+    int displayLevel, const char* displayName, const BMK_advancedParams_t* adv) 
+{
+    size_t const blockSize = ((adv->blockSize>=32 && (adv->mode != BMK_decodeOnly)) ? adv->blockSize : srcSize) + (!srcSize); /* avoid div by 0 */
+    BMK_return_t results;
+    size_t const loadedCompressedSize = srcSize;
+    size_t cSize = 0;
+    double ratio = 0.;
+    U32 nbBlocks;
+
+    if(!ctx || !dctx) 
+        EXM_THROW(31, BMK_return_t, "error: passed in null context");
+
+    /* init */
+    if (strlen(displayName)>17) displayName += strlen(displayName)-17;   /* display last 17 characters */
+    if (adv->mode == BMK_decodeOnly) {  /* benchmark only decompression : source must be already compressed */
+        const char* srcPtr = (const char*)srcBuffer;
+        U64 totalDSize64 = 0;
+        U32 fileNb;
+        for (fileNb=0; fileNb<nbFiles; fileNb++) {
+            U64 const fSize64 = ZSTD_findDecompressedSize(srcPtr, fileSizes[fileNb]);
+            if (fSize64==0) EXM_THROW(32, BMK_return_t, "Impossible to determine original size ");
+            totalDSize64 += fSize64;
+            srcPtr += fileSizes[fileNb];
+        }
+        {   size_t const decodedSize = (size_t)totalDSize64;
+            free(resultBuffer);
+            resultBuffer = malloc(decodedSize);
+            if (!resultBuffer) { 
+                EXM_THROW(33, BMK_return_t, "not enough memory"); 
+            }
+            if (totalDSize64 > decodedSize) {
+                free(resultBuffer); 
+                EXM_THROW(32, BMK_return_t, "original size is too large");   /* size_t overflow */
+            }
+            cSize = srcSize;
+            srcSize = decodedSize;
+            ratio = (double)srcSize / (double)cSize;
+        }   
+    }
+
+    /* Init data blocks  */
+    {   const char* srcPtr = (const char*)srcBuffer;
+        char* cPtr = (char*)compressedBuffer;
+        char* resPtr = (char*)resultBuffer;
+        U32 fileNb;
+        for (nbBlocks=0, fileNb=0; fileNb<nbFiles; fileNb++) {
+            size_t remaining = fileSizes[fileNb];
+            U32 const nbBlocksforThisFile = (adv->mode == BMK_decodeOnly) ? 1 : (U32)((remaining + (blockSize-1)) / blockSize);
+            U32 const blockEnd = nbBlocks + nbBlocksforThisFile;
+            for ( ; nbBlocks<blockEnd; nbBlocks++) {
+                size_t const thisBlockSize = MIN(remaining, blockSize);
+                srcPtrs[nbBlocks] = (const void*)srcPtr;
+                srcSizes[nbBlocks] = thisBlockSize;
+                cPtrs[nbBlocks] = (void*)cPtr;
+                cSizes[nbBlocks] = (adv->mode == BMK_decodeOnly) ? thisBlockSize : ZSTD_compressBound(thisBlockSize);
+                resPtrs[nbBlocks] = (void*)resPtr;
+                resSizes[nbBlocks] = (adv->mode == BMK_decodeOnly) ? (size_t) ZSTD_findDecompressedSize(srcPtr, thisBlockSize) : thisBlockSize;
+                srcPtr += thisBlockSize;
+                cPtr += cSizes[nbBlocks];
+                resPtr += thisBlockSize;
+                remaining -= thisBlockSize;
+            }   
+        }   
+    }
+
+    /* warmimg up memory */
+    if (adv->mode == BMK_decodeOnly) {
+        memcpy(compressedBuffer, srcBuffer, loadedCompressedSize);
+    } else {
+        RDG_genBuffer(compressedBuffer, maxCompressedSize, 0.10, 0.50, 1);
+    }
+
+    /* Bench */
+    {   
+        U64 const crcOrig = (adv->mode == BMK_decodeOnly) ? 0 : XXH64(srcBuffer, srcSize, 0);
+#       define NB_MARKS 4
+        const char* const marks[NB_MARKS] = { " |", " /", " =",  "\\" };
+        U32 markNb = 0;
+        DISPLAYLEVEL(2, "\r%79s\r", "");
+
+        DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->\r", marks[markNb], displayName, (U32)srcSize);
+        {
+            BMK_initCCtxArgs cctxprep;
+            BMK_initDCtxArgs dctxprep;
+            cctxprep.ctx = ctx;
+            cctxprep.dictBuffer = dictBuffer;
+            cctxprep.dictBufferSize = dictBufferSize;
+            cctxprep.cLevel = cLevel;
+            cctxprep.comprParams = comprParams;
+            cctxprep.adv = adv;
+            dctxprep.dctx = dctx;
+            dctxprep.dictBuffer = dictBuffer;
+            dctxprep.dictBufferSize = dictBufferSize;
+            if(adv->loopMode == BMK_timeMode) {
+                BMK_customTimedReturn_t intermediateResultCompress;
+                BMK_customTimedReturn_t intermediateResultDecompress;
+                if(adv->mode == BMK_compressOnly) {
+                    intermediateResultCompress.completed = 0;
+                    intermediateResultDecompress.completed = 1;
+                } else if (adv->mode == BMK_decodeOnly) {
+                    intermediateResultCompress.completed = 1;
+                    intermediateResultDecompress.completed = 0;
+                } else { /* both */
+                    intermediateResultCompress.completed = 0;
+                    intermediateResultDecompress.completed = 0;
+                }
+                while(!(intermediateResultCompress.completed && intermediateResultDecompress.completed)) {
+                    if(!intermediateResultCompress.completed) { 
+                        intermediateResultCompress = BMK_benchFunctionTimed(timeStateCompress, &local_defaultCompress, (void*)ctx, &local_initCCtx, (void*)&cctxprep,
+                        nbBlocks, srcPtrs, srcSizes, cPtrs, cSizes);
+                        if(intermediateResultCompress.result.error) {
+                            results.error = intermediateResultCompress.result.error;
+                            return results;
+                        }
+                        ratio = (double)(srcSize / intermediateResultCompress.result.result.sumOfReturn);
+                        {   
+                            int const ratioAccuracy = (ratio < 10.) ? 3 : 2;
+                            double const compressionSpeed = ((double)srcSize / intermediateResultCompress.result.result.nanoSecPerRun) * 1000;
                             int const cSpeedAccuracy = (compressionSpeed < 10.) ? 2 : 1;
-                double const decompressionSpeed = ((double)srcSize / fastestD) * 1000;
                             results.result.cSpeed = compressionSpeed * 1000000;
+                            results.result.cSize = intermediateResultCompress.result.result.sumOfReturn;
+                            ratio = (double)srcSize / results.result.cSize;
+                            markNb = (markNb+1) % NB_MARKS;
+                            DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.*f),%6.*f MB/s\r",
+                                    marks[markNb], displayName, (U32)srcSize, (U32)results.result.cSize,
+                                    ratioAccuracy, ratio,
+                                    cSpeedAccuracy, compressionSpeed);
+                        }
+                    }
+
+                    if(!intermediateResultDecompress.completed) {
+                        intermediateResultDecompress = BMK_benchFunctionTimed(timeStateDecompress, &local_defaultDecompress, (void*)(dctx), &local_initDCtx, (void*)&dctxprep,
+                        nbBlocks, (const void* const*)cPtrs, cSizes, resPtrs, resSizes);
+                        if(intermediateResultDecompress.result.error) {
+                            results.error = intermediateResultDecompress.result.error;
+                            return results;
+                        }
+      
+                        {   
+                            int const ratioAccuracy = (ratio < 10.) ? 3 : 2;
+                            double const compressionSpeed = results.result.cSpeed / 1000000;
+                            int const cSpeedAccuracy = (compressionSpeed < 10.) ? 2 : 1;
+                            double const decompressionSpeed = ((double)srcSize / intermediateResultDecompress.result.result.nanoSecPerRun) * 1000;
                             results.result.dSpeed = decompressionSpeed * 1000000;
+                            markNb = (markNb+1) % NB_MARKS;
                             DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.*f),%6.*f MB/s ,%6.1f MB/s \r",
-                        marks[markNb], displayName, (U32)srcSize, (U32)cSize,
+                                    marks[markNb], displayName, (U32)srcSize, (U32)results.result.cSize,
                                     ratioAccuracy, ratio,
                                     cSpeedAccuracy, compressionSpeed,
                                     decompressionSpeed);
                         }
+                    }
+                }
+
+            } else {
+                if(adv->mode != BMK_decodeOnly) {
+                    BMK_customReturn_t compressionResults = BMK_benchFunction(&local_defaultCompress, (void*)ctx, &local_initCCtx, (void*)&cctxprep,
+                        nbBlocks, srcPtrs, srcSizes, cPtrs, cSizes, adv->nbSeconds);
+                    if(compressionResults.error) {
+                        results.error = compressionResults.error;
+                        return results;
+                    }
+                    if(compressionResults.result.nanoSecPerRun == 0) {
+                        results.result.cSpeed = 0;
+                    } else {
+                        results.result.cSpeed = (double)srcSize / compressionResults.result.nanoSecPerRun * TIMELOOP_NANOSEC;
+                    }
+                    results.result.cSize = compressionResults.result.sumOfReturn;
+                    {   
+                        int const ratioAccuracy = (ratio < 10.) ? 3 : 2;
+                        double const compressionSpeed = results.result.cSpeed / 1000000;
+                        int const cSpeedAccuracy = (compressionSpeed < 10.) ? 2 : 1;
+                        ratio = (double)srcSize / results.result.cSize;
+                        markNb = (markNb+1) % NB_MARKS;
+                        DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.*f),%6.*f MB/s\r",
+                                marks[markNb], displayName, (U32)srcSize, (U32)results.result.cSize,
+                                ratioAccuracy, ratio,
+                                cSpeedAccuracy, compressionSpeed);
+                    }
+                }
+                if(adv->mode != BMK_compressOnly) {
+                    BMK_customReturn_t decompressionResults = BMK_benchFunction(
+                        &local_defaultDecompress, (void*)(dctx),
+                        &local_initDCtx, (void*)&dctxprep, nbBlocks,
+                        (const void* const*)cPtrs, cSizes, resPtrs, resSizes, 
+                        adv->nbSeconds);
+                    if(decompressionResults.error) {
+                        results.error = decompressionResults.error;
+                        return results;
+                    }
+                    if(decompressionResults.result.nanoSecPerRun == 0) {
+                        results.result.dSpeed = 0;
+                    } else {
+                        results.result.dSpeed = (double)srcSize / decompressionResults.result.nanoSecPerRun * TIMELOOP_NANOSEC;
+                    }
+                    {   int const ratioAccuracy = (ratio < 10.) ? 3 : 2;
+                        double const compressionSpeed = results.result.cSpeed / 1000000;
+                        int const cSpeedAccuracy = (compressionSpeed < 10.) ? 2 : 1;
+                        double const decompressionSpeed = ((double)srcSize / decompressionResults.result.nanoSecPerRun) * 1000;
+                        results.result.dSpeed = decompressionSpeed * 1000000;
+                        markNb = (markNb+1) % NB_MARKS;
+                        DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.*f),%6.*f MB/s ,%6.1f MB/s \r",
+                                marks[markNb], displayName, (U32)srcSize, (U32)results.result.cSize,
+                                ratioAccuracy, ratio,
+                                cSpeedAccuracy, compressionSpeed,
+                                decompressionSpeed);
+                    }
+                }
+            }
+        }
 
         /* CRC Checking */
         {   U64 const crcCheck = XXH64(resultBuffer, srcSize, 0);
-                if (!g_decodeOnly && (crcOrig!=crcCheck)) {
+            /* adv->mode == 0 -> compress + decompress */
+            if ((adv->mode == BMK_both) && (crcOrig!=crcCheck)) {
                 size_t u;
                 DISPLAY("!!! WARNING !!! %14s : Invalid Checksum : %x != %x   \n", displayName, (unsigned)crcOrig, (unsigned)crcCheck);
                 for (u=0; u<srcSize; u++) {
@@ -457,8 +652,8 @@ BMK_return_t BMK_benchMem(const void* srcBuffer, size_t srcSize,
                         size_t bacc = 0;
                         DISPLAY("Decoding error at pos %u ", (U32)u);
                         for (segNb = 0; segNb < nbBlocks; segNb++) {
-                                if (bacc + blockTable[segNb].srcSize > u) break;
+                            if (bacc + srcSizes[segNb] > u) break;
-                                bacc += blockTable[segNb].srcSize;
+                            bacc += srcSizes[segNb];
                         }
                         pos = (U32)(u - bacc);
                         bNb = pos / (128 KB);
@@ -480,50 +675,119 @@ BMK_return_t BMK_benchMem(const void* srcBuffer, size_t srcSize,
                     }
                     if (u==srcSize-1) {  /* should never happen */
                         DISPLAY("no difference detected\n");
-                    }   }
+                    }   
-                    break;
+                }
-            }   }   /* CRC Checking */
+            }   
-#endif
+        }   /* CRC Checking */
-        }   /* for (testNb = 1; testNb <= (g_nbSeconds + !g_nbSeconds); testNb++) */
 
     if (displayLevel == 1) {   /* hidden display mode -q, used by python speed benchmark */
-            double const cSpeed = ((double)srcSize / fastestC) * 1000;
+        double const cSpeed = results.result.cSpeed / 1000000;
-            double const dSpeed = ((double)srcSize / fastestD) * 1000;
+        double const dSpeed = results.result.dSpeed / 1000000;
-            if (g_additionalParam)
+        if (adv->additionalParam) {
-                DISPLAY("-%-3i%11i (%5.3f) %6.2f MB/s %6.1f MB/s  %s (param=%d)\n", cLevel, (int)cSize, ratio, cSpeed, dSpeed, displayName, g_additionalParam);
+            DISPLAY("-%-3i%11i (%5.3f) %6.2f MB/s %6.1f MB/s  %s (param=%d)\n", cLevel, (int)cSize, ratio, cSpeed, dSpeed, displayName, adv->additionalParam);
-            else
+        } else {
             DISPLAY("-%-3i%11i (%5.3f) %6.2f MB/s %6.1f MB/s  %s\n", cLevel, (int)cSize, ratio, cSpeed, dSpeed, displayName);
         }
+    }
     DISPLAYLEVEL(2, "%2i#\n", cLevel);
     }   /* Bench */
-
-    /* clean up */
-    free(blockTable);
-    free(compressedBuffer);
-    free(resultBuffer);
-    results.errorCode = 0;
     return results;
 }
 
-static void BMK_benchMemCtxless(const void* srcBuffer, size_t srcSize,
+BMK_return_t BMK_benchMemAdvanced(const void* srcBuffer, size_t srcSize,
                         const size_t* fileSizes, unsigned nbFiles,
-                        int cLevel, const ZSTD_compressionParameters* const comprParams,
+                        const int cLevel, const ZSTD_compressionParameters* comprParams,
                         const void* dictBuffer, size_t dictBufferSize,
-                         int displayLevel, const char* displayName) 
+                        ZSTD_CCtx* ctx, ZSTD_DCtx* dctx,
+                        int displayLevel, const char* displayName, const BMK_advancedParams_t* adv)
+
 {
-    ZSTD_CCtx* ctx = ZSTD_createCCtx();
+    size_t const blockSize = ((adv->blockSize>=32 && (adv->mode != BMK_decodeOnly)) ? adv->blockSize : srcSize) + (!srcSize) /* avoid div by 0 */ ;
-    ZSTD_DCtx* dctx = ZSTD_createDCtx();
+    U32 const maxNbBlocks = (U32) ((srcSize + (blockSize-1)) / blockSize) + nbFiles;
-    if(ctx == NULL || dctx == NULL) {
+
-        EXM_THROW(12, "not enough memory for contexts");
+    /* these are the blockTable parameters, just split up */
+    const void ** const srcPtrs = (const void** const)malloc(maxNbBlocks * sizeof(void*));
+    size_t* const srcSizes = (size_t* const)malloc(maxNbBlocks * sizeof(size_t));
+
+    void ** const cPtrs = (void** const)malloc(maxNbBlocks * sizeof(void*));
+    size_t* const cSizes = (size_t* const)malloc(maxNbBlocks * sizeof(size_t));
+
+    void ** const resPtrs = (void** const)malloc(maxNbBlocks * sizeof(void*));
+    size_t* const resSizes = (size_t* const)malloc(maxNbBlocks * sizeof(size_t));
+
+    const size_t maxCompressedSize = ZSTD_compressBound(srcSize) + (maxNbBlocks * 1024);   /* add some room for safety */
+    void* compressedBuffer = malloc(maxCompressedSize);
+    void* resultBuffer = malloc(srcSize);
+    BMK_timedFnState_t* timeStateCompress = BMK_createTimeState(adv->nbSeconds);
+    BMK_timedFnState_t* timeStateDecompress = BMK_createTimeState(adv->nbSeconds);
+
+    BMK_return_t results;
+    int allocationincomplete = !compressedBuffer || !resultBuffer ||  
+        !srcPtrs || !srcSizes || !cPtrs || !cSizes || !resPtrs || !resSizes;
+    if (!allocationincomplete) {
+        results = BMK_benchMemAdvancedNoAlloc(srcPtrs, srcSizes, cPtrs, cSizes,
+            resPtrs, resSizes, resultBuffer, compressedBuffer, maxCompressedSize, timeStateCompress, timeStateDecompress,
+            srcBuffer, srcSize, fileSizes, nbFiles, cLevel, comprParams,
+            dictBuffer, dictBufferSize, ctx, dctx, displayLevel, displayName, adv);
     }
-    BMK_benchMem(srcBuffer, srcSize, 
+    /* clean up */
+    BMK_freeTimeState(timeStateCompress);
+    BMK_freeTimeState(timeStateDecompress);
+    free(compressedBuffer);
+    free(resultBuffer);
+
+    free((void*)srcPtrs); 
+    free(srcSizes); 
+    free(cPtrs); 
+    free(cSizes);
+    free(resPtrs);
+    free(resSizes);
+
+    if(allocationincomplete) {
+        EXM_THROW(31, BMK_return_t, "allocation error : not enough memory");
+    }
+    results.error = 0;
+    return results;
+}
+
+BMK_return_t BMK_benchMem(const void* srcBuffer, size_t srcSize,
+                        const size_t* fileSizes, unsigned nbFiles,
+                        const int cLevel, const ZSTD_compressionParameters* comprParams,
+                        const void* dictBuffer, size_t dictBufferSize,
+                        ZSTD_CCtx* ctx, ZSTD_DCtx* dctx,
+                        int displayLevel, const char* displayName) {
+
+    const BMK_advancedParams_t adv = BMK_initAdvancedParams();
+    return BMK_benchMemAdvanced(srcBuffer, srcSize,
                                 fileSizes, nbFiles,
                                 cLevel, comprParams,
                                 dictBuffer, dictBufferSize,
                                 ctx, dctx,
-                displayLevel, displayName);
+                                displayLevel, displayName, &adv);
+}
+
+static BMK_return_t BMK_benchMemCtxless(const void* srcBuffer, size_t srcSize,
+                        const size_t* fileSizes, unsigned nbFiles, 
+                        int cLevel, const ZSTD_compressionParameters* const comprParams,
+                        const void* dictBuffer, size_t dictBufferSize,
+                        int displayLevel, const char* displayName, 
+                        const BMK_advancedParams_t* const adv) 
+{
+    BMK_return_t res;
+    ZSTD_CCtx* ctx = ZSTD_createCCtx();
+    ZSTD_DCtx* dctx = ZSTD_createDCtx();
+    if(ctx == NULL || dctx == NULL) {
+        EXM_THROW(12, BMK_return_t, "not enough memory for contexts");
+    }
+    res = BMK_benchMemAdvanced(srcBuffer, srcSize, 
+                fileSizes, nbFiles, 
+                cLevel, comprParams, 
+                dictBuffer, dictBufferSize, 
+                ctx, dctx, 
+                displayLevel, displayName, adv);
     ZSTD_freeCCtx(ctx);
     ZSTD_freeDCtx(dctx);
+    return res;
 }
 
 static size_t BMK_findMaxMem(U64 requiredMem)
@@ -544,44 +808,43 @@ static size_t BMK_findMaxMem(U64 requiredMem)
     return (size_t)(requiredMem);
 }
 
-/* returns average stats over all range [cLevel, cLevelLast] */
+static BMK_return_t BMK_benchCLevel(const void* srcBuffer, size_t benchedSize,
-static void BMK_benchCLevel(const void* srcBuffer, size_t benchedSize,
                             const size_t* fileSizes, unsigned nbFiles,
-                            const int cLevel, const int cLevelLast, const ZSTD_compressionParameters* comprParams,
+                            const int cLevel, const ZSTD_compressionParameters* comprParams,
                             const void* dictBuffer, size_t dictBufferSize,
-                            int displayLevel, const char* displayName)
+                            int displayLevel, const char* displayName,
+                            BMK_advancedParams_t const * const adv)
 {
-    int l;
+    BMK_return_t res;
 
     const char* pch = strrchr(displayName, '\\'); /* Windows */
+
     if (!pch) pch = strrchr(displayName, '/'); /* Linux */
     if (pch) displayName = pch+1;
 
-    if (g_realTime) {
+    if (adv->realTime) {
         DISPLAYLEVEL(2, "Note : switching to real-time priority \n");
         SET_REALTIME_PRIORITY;
     }
 
-    if (displayLevel == 1 && !g_additionalParam)
+    if (displayLevel == 1 && !adv->additionalParam)
-        DISPLAY("bench %s %s: input %u bytes, %u seconds, %u KB blocks\n", ZSTD_VERSION_STRING, ZSTD_GIT_COMMIT_STRING, (U32)benchedSize, g_nbSeconds, (U32)(g_blockSize>>10));
+        DISPLAY("bench %s %s: input %u bytes, %u seconds, %u KB blocks\n", ZSTD_VERSION_STRING, ZSTD_GIT_COMMIT_STRING, (U32)benchedSize, adv->nbSeconds, (U32)(adv->blockSize>>10));
 
-    for (l=cLevel; l <= cLevelLast; l++) {
+    res = BMK_benchMemCtxless(srcBuffer, benchedSize,
-        if (l==0) continue;  /* skip level 0 */
-        BMK_benchMemCtxless(srcBuffer, benchedSize,
             fileSizes, nbFiles, 
-                    l, comprParams, 
+            cLevel, comprParams, 
             dictBuffer, dictBufferSize, 
-                    displayLevel, displayName);
+            displayLevel, displayName, 
-    }
+            adv);
 
-    return;
+    return res;
 }
 
 
 /*! BMK_loadFiles() :
  *  Loads `buffer` with content of files listed within `fileNamesTable`.
  *  At most, fills `buffer` entirely. */
-static void BMK_loadFiles(void* buffer, size_t bufferSize,
+static int BMK_loadFiles(void* buffer, size_t bufferSize,
                           size_t* fileSizes, const char* const * const fileNamesTable, 
                           unsigned nbFiles, int displayLevel)
 {
@@ -601,44 +864,65 @@ static void BMK_loadFiles(void* buffer, size_t bufferSize,
             continue;
         }
         f = fopen(fileNamesTable[n], "rb");
-        if (f==NULL) EXM_THROW(10, "impossible to open file %s", fileNamesTable[n]);
+        if (f==NULL) EXM_THROW_INT(10, "impossible to open file %s", fileNamesTable[n]);
         DISPLAYUPDATE(2, "Loading %s...       \r", fileNamesTable[n]);
         if (fileSize > bufferSize-pos) fileSize = bufferSize-pos, nbFiles=n;   /* buffer too small - stop after this file */
         { size_t const readSize = fread(((char*)buffer)+pos, 1, (size_t)fileSize, f);
-          if (readSize != (size_t)fileSize) EXM_THROW(11, "could not read %s", fileNamesTable[n]);
+          if (readSize != (size_t)fileSize) EXM_THROW_INT(11, "could not read %s", fileNamesTable[n]);
           pos += readSize; }
         fileSizes[n] = (size_t)fileSize;
         totalSize += (size_t)fileSize;
         fclose(f);
     }
 
-    if (totalSize == 0) EXM_THROW(12, "no data to bench");
+    if (totalSize == 0) EXM_THROW_INT(12, "no data to bench");
+    return 0;
 }
 
-static void BMK_benchFileTable(const char* const * const fileNamesTable, unsigned const nbFiles,
+BMK_return_t BMK_benchFilesAdvanced(const char* const * const fileNamesTable, unsigned const nbFiles,
-                               const char* const dictFileName, int const cLevel, int const cLevelLast,
+                               const char* const dictFileName, int const cLevel, 
-                               const ZSTD_compressionParameters* const compressionParams, int displayLevel)
+                               const ZSTD_compressionParameters* const compressionParams, 
+                               int displayLevel, const BMK_advancedParams_t * const adv)
 {
-    void* srcBuffer;
+    void* srcBuffer = NULL;
     size_t benchedSize;
     void* dictBuffer = NULL;
     size_t dictBufferSize = 0;
-    size_t* const fileSizes = (size_t*)malloc(nbFiles * sizeof(size_t));
+    size_t* fileSizes = NULL;
+    BMK_return_t res;
     U64 const totalSizeToLoad = UTIL_getTotalFileSize(fileNamesTable, nbFiles);
 
-    if (!fileSizes) EXM_THROW(12, "not enough memory for fileSizes");
+    if(!nbFiles) {
+        EXM_THROW(14, BMK_return_t, "No Files to Benchmark");
+    }
 
+    if (cLevel > ZSTD_maxCLevel()) {
+        EXM_THROW(15, BMK_return_t, "Invalid Compression Level");
+    }
+
+    fileSizes = (size_t*)calloc(nbFiles, sizeof(size_t));
+    if (!fileSizes) EXM_THROW(12, BMK_return_t, "not enough memory for fileSizes");
     /* Load dictionary */
     if (dictFileName != NULL) {
         U64 const dictFileSize = UTIL_getFileSize(dictFileName);
-        if (dictFileSize > 64 MB)
+        if (dictFileSize > 64 MB) {
-            EXM_THROW(10, "dictionary file %s too large", dictFileName);
+            free(fileSizes);
+            EXM_THROW(10, BMK_return_t, "dictionary file %s too large", dictFileName);
+        }
         dictBufferSize = (size_t)dictFileSize;
         dictBuffer = malloc(dictBufferSize);
-        if (dictBuffer==NULL)
+        if (dictBuffer==NULL) {
-            EXM_THROW(11, "not enough memory for dictionary (%u bytes)",
+            free(fileSizes);
+            EXM_THROW(11, BMK_return_t, "not enough memory for dictionary (%u bytes)",
                             (U32)dictBufferSize);
-        BMK_loadFiles(dictBuffer, dictBufferSize, fileSizes, &dictFileName, 1, displayLevel);
+        }
+        {
+            int errorCode = BMK_loadFiles(dictBuffer, dictBufferSize, fileSizes, &dictFileName, 1, displayLevel);
+            if(errorCode) {
+                res.error = errorCode;
+                goto _cleanUp;
+            }
+        }
     }
 
     /* Memory allocation & restrictions */
@@ -647,96 +931,80 @@ static void BMK_benchFileTable(const char* const * const fileNamesTable, unsigne
     if (benchedSize < totalSizeToLoad)
         DISPLAY("Not enough memory; testing %u MB only...\n", (U32)(benchedSize >> 20));
     srcBuffer = malloc(benchedSize);
-    if (!srcBuffer) EXM_THROW(12, "not enough memory");
+    if (!srcBuffer) {
-
+        free(dictBuffer);
-    /* Load input buffer */
+        free(fileSizes);
-    BMK_loadFiles(srcBuffer, benchedSize, fileSizes, fileNamesTable, nbFiles, displayLevel);
+        EXM_THROW(12, BMK_return_t, "not enough memory");
-
-    /* Bench */
-    if (g_separateFiles) {
-        const BYTE* srcPtr = (const BYTE*)srcBuffer;
-        U32 fileNb;
-        BMK_result_t* resultarray = (BMK_result_t*)malloc(sizeof(BMK_result_t) * nbFiles);
-        if(resultarray == NULL) EXM_THROW(12, "not enough memory");
-        for (fileNb=0; fileNb<nbFiles; fileNb++) {
-            size_t const fileSize = fileSizes[fileNb];
-            BMK_benchCLevel(srcPtr, fileSize,
-                            fileSizes+fileNb, 1, 
-                            cLevel, cLevelLast, compressionParams,
-                            dictBuffer, dictBufferSize, 
-                            displayLevel, fileNamesTable[fileNb]);
-            srcPtr += fileSize;
     }
 
-    } else {
+    /* Load input buffer */
+    {
+        int errorCode = BMK_loadFiles(srcBuffer, benchedSize, fileSizes, fileNamesTable, nbFiles, displayLevel);
+        if(errorCode) {
+            res.error = errorCode;
+            goto _cleanUp;
+        }
+    }
+    /* Bench */
+    {
         char mfName[20] = {0};
         snprintf (mfName, sizeof(mfName), " %u files", nbFiles);
-        {   const char* const displayName = (nbFiles > 1) ? mfName : fileNamesTable[0];
+        {   
-            BMK_benchCLevel(srcBuffer, benchedSize, 
+            const char* const displayName = (nbFiles > 1) ? mfName : fileNamesTable[0];
+            res = BMK_benchCLevel(srcBuffer, benchedSize, 
                             fileSizes, nbFiles, 
-                            cLevel, cLevelLast, compressionParams,
+                            cLevel, compressionParams,
                             dictBuffer, dictBufferSize, 
-                            displayLevel, displayName);
+                            displayLevel, displayName, 
+                            adv);
     }   }
 
-    /* clean up */
+_cleanUp:
     free(srcBuffer);
     free(dictBuffer);
     free(fileSizes);
+    return res;
 }
 
 
-static void BMK_syntheticTest(int cLevel, int cLevelLast, double compressibility,
+BMK_return_t BMK_syntheticTest(int cLevel, double compressibility,
                               const ZSTD_compressionParameters* compressionParams,
-                              int displayLevel)
+                              int displayLevel, const BMK_advancedParams_t * const adv)
 {
     char name[20] = {0};
     size_t benchedSize = 10000000;
-    void* const srcBuffer = malloc(benchedSize);
+    void* srcBuffer;
+    BMK_return_t res;
+
+    if (cLevel > ZSTD_maxCLevel()) {
+        EXM_THROW(15, BMK_return_t, "Invalid Compression Level");
+    }
 
     /* Memory allocation */
-    if (!srcBuffer) EXM_THROW(21, "not enough memory");
+    srcBuffer = malloc(benchedSize);
+    if (!srcBuffer) EXM_THROW(21, BMK_return_t, "not enough memory");
 
     /* Fill input buffer */
     RDG_genBuffer(srcBuffer, benchedSize, compressibility, 0.0, 0);
 
     /* Bench */
     snprintf (name, sizeof(name), "Synthetic %2u%%", (unsigned)(compressibility*100));
-    BMK_benchCLevel(srcBuffer, benchedSize, 
+    res = BMK_benchCLevel(srcBuffer, benchedSize, 
                     &benchedSize, 1, 
-                    cLevel, cLevelLast, compressionParams, 
+                    cLevel, compressionParams, 
                     NULL, 0, 
-                    displayLevel, name);
+                    displayLevel, name, adv);
 
     /* clean up */
     free(srcBuffer);
+
+    return res;
 }
 
-
+BMK_return_t BMK_benchFiles(const char* const * const fileNamesTable, unsigned const nbFiles,
-static void BMK_benchFilesFull(const char** fileNamesTable, unsigned nbFiles,
+                   const char* const dictFileName, 
-                   const char* dictFileName, 
+                   int const cLevel, const ZSTD_compressionParameters* const compressionParams, 
-                   int cLevel, int cLevelLast, 
-                   const ZSTD_compressionParameters* compressionParams, int displayLevel)
-{
-    double const compressibility = (double)g_compressibilityDefault / 100;
-
-    if (cLevel > ZSTD_maxCLevel()) cLevel = ZSTD_maxCLevel();
-    if (cLevelLast > ZSTD_maxCLevel()) cLevelLast = ZSTD_maxCLevel();
-    if (cLevelLast < cLevel) cLevelLast = cLevel;
-    if (cLevelLast > cLevel)
-        DISPLAYLEVEL(2, "Benchmarking levels from %d to %d\n", cLevel, cLevelLast);
-
-    if (nbFiles == 0)
-        BMK_syntheticTest(cLevel, cLevelLast, compressibility, compressionParams, displayLevel);
-    else
-        BMK_benchFileTable(fileNamesTable, nbFiles, dictFileName, cLevel, cLevelLast, compressionParams, displayLevel);
-}
-
-int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles,
-                   const char* dictFileName,
-                   int cLevel, int cLevelLast,
-                   const ZSTD_compressionParameters* compressionParams,
                    int displayLevel) {
-    BMK_benchFilesFull(fileNamesTable, nbFiles, dictFileName, cLevel, cLevelLast, compressionParams, displayLevel);
+    const BMK_advancedParams_t adv = BMK_initAdvancedParams();
-    return 0;
+    return BMK_benchFilesAdvanced(fileNamesTable, nbFiles, dictFileName, cLevel, compressionParams, displayLevel, &adv);
 }

programs/bench.h

@@ -19,25 +19,121 @@ extern "C" {
 #define ZSTD_STATIC_LINKING_ONLY   /* ZSTD_compressionParameters */
 #include "zstd.h"     /* ZSTD_compressionParameters */
 
+/* Creates a struct of type typeName with an int type .error field
+ * and a .result field of some baseType. Functions with return
+ * typeName pass a successful result with .error = 0 and .result
+ * with the intended result, while returning an error will result
+ * in .error != 0. 
+ */
+#define ERROR_STRUCT(baseType, typeName) typedef struct { \
+    baseType result; \
+    int error;       \
+} typeName
+
 typedef struct {
     size_t cSize;
     double cSpeed;   /* bytes / sec */
     double dSpeed;
 } BMK_result_t;
 
-/* 0 = no Error */
+ERROR_STRUCT(BMK_result_t, BMK_return_t);
-typedef struct {
-	int errorCode;
-	BMK_result_t result;
-} BMK_return_t;
 
 /* called in cli */
-int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles, const char* dictFileName,
+/* Loads files in fileNamesTable into memory, as well as a dictionary 
-                   int cLevel, int cLevelLast, const ZSTD_compressionParameters* compressionParams, 
+ * from dictFileName, and then uses benchMem */
+/* fileNamesTable - name of files to benchmark
+ * nbFiles - number of files (size of fileNamesTable), must be > 0
+ * dictFileName - name of dictionary file to load
+ * cLevel - compression level to benchmark, errors if invalid
+ * compressionParams - basic compression Parameters
+ * displayLevel - what gets printed
+ *      0 : no display;   
+ *      1 : errors;   
+ *      2 : + result + interaction + warnings;   
+ *      3 : + progression;   
+ *      4 : + information
+ * return 
+ *      .error will give a nonzero error value if an error has occured
+ *      .result - if .error = 0, .result will return the time taken to compression speed
+ *          (.cSpeed), decompression speed (.dSpeed), and compressed size (.cSize) of the original
+ *          file
+ */
+BMK_return_t BMK_benchFiles(const char* const * const fileNamesTable, unsigned const nbFiles,
+                   const char* const dictFileName, 
+                   int const cLevel, const ZSTD_compressionParameters* const compressionParams, 
                    int displayLevel);
 
+typedef enum {
+    BMK_timeMode = 0,
+    BMK_iterMode = 1
+} BMK_loopMode_t;
+
+typedef enum {
+    BMK_both = 0,
+    BMK_decodeOnly = 1,
+    BMK_compressOnly = 2
+} BMK_mode_t;
+
+typedef struct {
+    BMK_mode_t mode;            /* 0: all, 1: compress only 2: decode only */
+    BMK_loopMode_t loopMode;    /* if loopmode, then nbSeconds = nbLoops */
+    unsigned nbSeconds;         /* default timing is in nbSeconds */
+    size_t blockSize;           /* Maximum allowable size of a block*/
+    unsigned nbWorkers;         /* multithreading */
+    unsigned realTime;          /* real time priority */
+    int additionalParam;        /* used by python speed benchmark */
+    unsigned ldmFlag;           /* enables long distance matching */
+    unsigned ldmMinMatch;       /* below: parameters for long distance matching, see zstd.1.md for meaning */
+    unsigned ldmHashLog; 
+    unsigned ldmBucketSizeLog;
+    unsigned ldmHashEveryLog;
+} BMK_advancedParams_t;
+
+/* returns default parameters used by nonAdvanced functions */
+BMK_advancedParams_t BMK_initAdvancedParams(void);
+
+/* See benchFiles for normal parameter uses and return, see advancedParams_t for adv */
+BMK_return_t BMK_benchFilesAdvanced(const char* const * const fileNamesTable, unsigned const nbFiles,
+                   const char* const dictFileName, 
+                   int const cLevel, const ZSTD_compressionParameters* const compressionParams, 
+                   int displayLevel, const BMK_advancedParams_t* const adv);
+
+/* called in cli */
+/* Generates a sample with datagen with the compressibility argument*/
+/* cLevel - compression level to benchmark, errors if invalid
+ * compressibility - determines compressibility of sample
+ * compressionParams - basic compression Parameters
+ * displayLevel - see benchFiles
+ * adv - see advanced_Params_t
+ * return 
+ *      .error will give a nonzero error value if an error has occured
+ *      .result - if .error = 0, .result will return the time taken to compression speed
+ *          (.cSpeed), decompression speed (.dSpeed), and compressed size (.cSize) of the original
+ *          file
+ */
+BMK_return_t BMK_syntheticTest(int cLevel, double compressibility,
+                              const ZSTD_compressionParameters* compressionParams,
+                              int displayLevel, const BMK_advancedParams_t * const adv);
+
 /* basic benchmarking function, called in paramgrill 
- * ctx, dctx must be valid */
+ * applies ZSTD_compress_generic() and ZSTD_decompress_generic() on data in srcBuffer
+ * with specific compression parameters specified by other arguments using benchFunction
+ * (cLevel, comprParams + adv in advanced Mode) */
+/* srcBuffer - data source, expected to be valid compressed data if in Decode Only Mode
+ * srcSize - size of data in srcBuffer
+ * cLevel - compression level  
+ * comprParams - basic compression parameters
+ * dictBuffer - a dictionary if used, null otherwise
+ * dictBufferSize - size of dictBuffer, 0 otherwise
+ * ctx - Compression Context (must be provided)
+ * dctx - Decompression Context (must be provided)
+ * diplayLevel - see BMK_benchFiles
+ * displayName - name used by display
+ * return
+ *      .error will give a nonzero value if an error has occured
+ *      .result - if .error = 0, will give the same results as benchFiles
+ *          but for the data stored in srcBuffer
+ */
 BMK_return_t BMK_benchMem(const void* srcBuffer, size_t srcSize,
                         const size_t* fileSizes, unsigned nbFiles,
                         const int cLevel, const ZSTD_compressionParameters* comprParams,
@@ -45,20 +141,82 @@ BMK_return_t BMK_benchMem(const void* srcBuffer, size_t srcSize,
                         ZSTD_CCtx* ctx, ZSTD_DCtx* dctx,
                         int displayLevel, const char* displayName);
 
-/* Set Parameters */
+/* See benchMem for normal parameter uses and return, see advancedParams_t for adv */
-void BMK_setNbSeconds(unsigned nbLoops);
+BMK_return_t BMK_benchMemAdvanced(const void* srcBuffer, size_t srcSize,
-void BMK_setBlockSize(size_t blockSize);
+                        const size_t* fileSizes, unsigned nbFiles,
-void BMK_setNbWorkers(unsigned nbWorkers);
+                        const int cLevel, const ZSTD_compressionParameters* comprParams,
-void BMK_setRealTime(unsigned priority);
+                        const void* dictBuffer, size_t dictBufferSize,
-void BMK_setNotificationLevel(unsigned level);
+                        ZSTD_CCtx* ctx, ZSTD_DCtx* dctx,
-void BMK_setSeparateFiles(unsigned separate);
+                        int displayLevel, const char* displayName,
-void BMK_setAdditionalParam(int additionalParam);
+                        const BMK_advancedParams_t* adv);
-void BMK_setDecodeOnlyMode(unsigned decodeFlag);
+
-void BMK_setLdmFlag(unsigned ldmFlag);
+typedef struct {
-void BMK_setLdmMinMatch(unsigned ldmMinMatch);
+    size_t sumOfReturn;    /* sum of return values */
-void BMK_setLdmHashLog(unsigned ldmHashLog);
+    U64 nanoSecPerRun;     /* time per iteration */
-void BMK_setLdmBucketSizeLog(unsigned ldmBucketSizeLog);
+} BMK_customResult_t;
-void BMK_setLdmHashEveryLog(unsigned ldmHashEveryLog);
+
+ERROR_STRUCT(BMK_customResult_t, BMK_customReturn_t);
+
+typedef size_t (*BMK_benchFn_t)(const void*, size_t, void*, size_t, void*);
+typedef size_t (*BMK_initFn_t)(void*);
+
+/* This function times the execution of 2 argument functions, benchFn and initFn  */
+
+/* benchFn - (*benchFn)(srcBuffers[i], srcSizes[i], dstBuffers[i], dstCapacities[i], benchPayload)
+ *      is run nbLoops times
+ * initFn - (*initFn)(initPayload) is run once per benchmark at the beginning. This argument can 
+ *          be NULL, in which case nothing is run.
+ * blockCount - number of blocks (size of srcBuffers, srcSizes, dstBuffers, dstCapacities)
+ * srcBuffers - an array of buffers to be operated on by benchFn
+ * srcSizes - an array of the sizes of above buffers
+ * dstBuffers - an array of buffers to be written into by benchFn
+ * dstCapacities - an array of the capacities of above buffers.
+ * nbLoops - defines number of times benchFn is run.
+ * return 
+ *      .error will give a nonzero value if ZSTD_isError() is nonzero for any of the return
+ *          of the calls to initFn and benchFn, or if benchFunction errors internally
+ *      .result - if .error = 0, then .result will contain the sum of all return values of 
+ *          benchFn on the first iteration through all of the blocks (.sumOfReturn) and also 
+ *          the time per run of benchFn (.nanoSecPerRun). For the former, this
+ *          is generally intended to be used on functions which return the # of bytes written 
+ *          into dstBuffer, hence this value will be the total amount of bytes written to 
+ *          dstBuffer.
+ */
+BMK_customReturn_t BMK_benchFunction(                        
+                        BMK_benchFn_t benchFn, void* benchPayload,
+                        BMK_initFn_t initFn, void* initPayload,
+                        size_t blockCount,
+                        const void* const * const srcBuffers, const size_t* srcSizes,
+                        void* const * const dstBuffers, const size_t* dstCapacities,
+                        unsigned nbLoops);
+
+
+/* state information needed to advance computation for benchFunctionTimed */
+typedef struct BMK_timeState_t BMK_timedFnState_t;
+/* initializes timeState object with desired number of seconds */
+BMK_timedFnState_t* BMK_createTimeState(unsigned nbSeconds);
+/* resets existing timeState object */
+void BMK_resetTimeState(BMK_timedFnState_t*, unsigned nbSeconds);
+/* deletes timeState object */
+void BMK_freeTimeState(BMK_timedFnState_t* state);
+
+typedef struct {
+    BMK_customReturn_t result;
+    int completed;
+} BMK_customTimedReturn_t;
+
+/* 
+ * Benchmarks custom functions like BMK_benchFunction(), but runs for nbSeconds seconds rather than a fixed number of loops
+ * arguments mostly the same other than BMK_benchFunction()
+ * Usage - benchFunctionTimed will return in approximately one second. Keep calling BMK_benchFunctionTimed() until the return's completed field = 1. 
+ * to continue updating intermediate result. Intermediate return values are returned by the function.
+ */
+BMK_customTimedReturn_t BMK_benchFunctionTimed(BMK_timedFnState_t* cont,
+    BMK_benchFn_t benchFn, void* benchPayload,
+    BMK_initFn_t initFn, void* initPayload,
+    size_t blockCount,
+    const void* const * const srcBlockBuffers, const size_t* srcBlockSizes,
+    void* const * const dstBlockBuffers, const size_t* dstBlockCapacities);
 
 #endif   /* BENCH_H_121279284357 */
 

programs/dibio.c

@@ -323,7 +323,8 @@ int DiB_trainFromFiles(const char* dictFileName, unsigned maxDictSize,
                                                            srcBuffer, sampleSizes, fs.nbSamples,
                                                            coverParams);
             if (!ZDICT_isError(dictSize)) {
-                DISPLAYLEVEL(2, "k=%u\nd=%u\nsteps=%u\n", coverParams->k, coverParams->d, coverParams->steps);
+                unsigned splitPercentage = (unsigned)(coverParams->splitPoint * 100);
+                DISPLAYLEVEL(2, "k=%u\nd=%u\nsteps=%u\nsplit=%u\n", coverParams->k, coverParams->d, coverParams->steps, splitPercentage);
             }
         } else {
             dictSize = ZDICT_trainFromBuffer_cover(dictBuffer, maxDictSize, srcBuffer,

programs/fileio.c

@@ -2017,21 +2017,25 @@ static int FIO_listFile(fileInfo_t* total, const char* inFileName, int displayLe
 }
 
 int FIO_listMultipleFiles(unsigned numFiles, const char** filenameTable, int displayLevel){
-
+    unsigned u;
-    if (!IS_CONSOLE(stdin)) {
+    for (u=0; u<numFiles;u++) {
+        if (!strcmp (filenameTable[u], stdinmark)) {
             DISPLAYOUT("zstd: --list does not support reading from standard input\n");
             return 1;
         }
+    }
 
     if (numFiles == 0) {
+        if (!IS_CONSOLE(stdin)) {
+            DISPLAYOUT("zstd: --list does not support reading from standard input\n");
+        }
         DISPLAYOUT("No files given\n");
-        return 0;
+        return 1;
     }
     if (displayLevel <= 2) {
         DISPLAYOUT("Frames  Skips  Compressed  Uncompressed  Ratio  Check  Filename\n");
     }
     {   int error = 0;
-        unsigned u;
         fileInfo_t total;
         memset(&total, 0, sizeof(total));
         total.usesCheck = 1;

programs/zstd.1

@@ -217,8 +217,9 @@ Split input files in blocks of size # (default: no split)
 A dictionary ID is a locally unique ID that a decoder can use to verify it is using the right dictionary\. By default, zstd will create a 4\-bytes random number ID\. It\'s possible to give a precise number instead\. Short numbers have an advantage : an ID < 256 will only need 1 byte in the compressed frame header, and an ID < 65536 will only need 2 bytes\. This compares favorably to 4 bytes default\. However, it\'s up to the dictionary manager to not assign twice the same ID to 2 different dictionaries\.
 .
 .TP
-\fB\-\-train\-cover[=k#,d=#,steps=#]\fR
+\fB\-\-train\-cover[=k#,d=#,steps=#,split=#]\fR
-Select parameters for the default dictionary builder algorithm named cover\. If \fId\fR is not specified, then it tries \fId\fR = 6 and \fId\fR = 8\. If \fIk\fR is not specified, then it tries \fIsteps\fR values in the range [50, 2000]\. If \fIsteps\fR is not specified, then the default value of 40 is used\. Requires that \fId\fR <= \fIk\fR\.
+Select parameters for the default dictionary builder algorithm named cover\. If \fId\fR is not specified, then it tries \fId\fR = 6 and \fId\fR = 8\. If \fIk\fR is not specified, then it tries \fIsteps\fR values in the range [50, 2000]\. If \fIsteps\fR is not specified, then the default value of 40 is used\. If \fIsplit\fR is not specified or \fIsplit\fR <= 0, then the default value of 100 is used\. If \fIsplit\fR is 100, all input samples are used for both training and testing
+to find optimal _d_ and _k_ to build dictionary.Requires that \fId\fR <= \fIk\fR\.
 .
 .IP
 Selects segments of size \fIk\fR with highest score to put in the dictionary\. The score of a segment is computed by the sum of the frequencies of all the subsegments of size \fId\fR\. Generally \fId\fR should be in the range [6, 8], occasionally up to 16, but the algorithm will run faster with d <= \fI8\fR\. Good values for \fIk\fR vary widely based on the input data, but a safe range is [2 * \fId\fR, 2000]\. Supports multithreading if \fBzstd\fR is compiled with threading support\.

programs/zstd.1.md

@@ -223,11 +223,12 @@ Compression of small files similar to the sample set will be greatly improved.
     This compares favorably to 4 bytes default.
     However, it's up to the dictionary manager to not assign twice the same ID to
     2 different dictionaries.
-* `--train-cover[=k#,d=#,steps=#]`:
+* `--train-cover[=k#,d=#,steps=#,split=#]`:
     Select parameters for the default dictionary builder algorithm named cover.
     If _d_ is not specified, then it tries _d_ = 6 and _d_ = 8.
     If _k_ is not specified, then it tries _steps_ values in the range [50, 2000].
     If _steps_ is not specified, then the default value of 40 is used.
+    If _split_ is not specified or split <= 0, then the default value of 100 is used.
     Requires that _d_ <= _k_.
 
     Selects segments of size _k_ with highest score to put in the dictionary.
@@ -237,6 +238,8 @@ Compression of small files similar to the sample set will be greatly improved.
     algorithm will run faster with d <= _8_.
     Good values for _k_ vary widely based on the input data, but a safe range is
     [2 * _d_, 2000].
+    If _split_ is 100, all input samples are used for both training and testing
+    to find optimal _d_ and _k_ to build dictionary.
     Supports multithreading if `zstd` is compiled with threading support.
 
     Examples:
@@ -249,6 +252,8 @@ Compression of small files similar to the sample set will be greatly improved.
 
     `zstd --train-cover=k=50 FILEs`
 
+    `zstd --train-cover=k=50,split=60 FILEs`
+
 * `--train-legacy[=selectivity=#]`:
     Use legacy dictionary builder algorithm with the given dictionary
     _selectivity_ (default: 9).

programs/zstdcli.c

@@ -32,7 +32,7 @@
 #include <errno.h>    /* errno */
 #include "fileio.h"   /* stdinmark, stdoutmark, ZSTD_EXTENSION */
 #ifndef ZSTD_NOBENCH
-#  include "bench.h"  /* BMK_benchFiles, BMK_SetNbSeconds */
+#  include "bench.h"  /* BMK_benchFiles */
 #endif
 #ifndef ZSTD_NODICT
 #  include "dibio.h"  /* ZDICT_cover_params_t, DiB_trainFromFiles() */
@@ -84,6 +84,7 @@ static U32 g_ldmMinMatch = 0;
 static U32 g_ldmHashEveryLog = LDM_PARAM_DEFAULT;
 static U32 g_ldmBucketSizeLog = LDM_PARAM_DEFAULT;
 
+#define DEFAULT_SPLITPOINT 1.0
 
 /*-************************************
 *  Display Macros
@@ -170,7 +171,7 @@ static int usage_advanced(const char* programName)
     DISPLAY( "\n");
     DISPLAY( "Dictionary builder : \n");
     DISPLAY( "--train ## : create a dictionary from a training set of files \n");
-    DISPLAY( "--train-cover[=k=#,d=#,steps=#] : use the cover algorithm with optional args\n");
+    DISPLAY( "--train-cover[=k=#,d=#,steps=#,split=#] : use the cover algorithm with optional args\n");
     DISPLAY( "--train-legacy[=s=#] : use the legacy algorithm with selectivity (default: %u)\n", g_defaultSelectivityLevel);
     DISPLAY( " -o file : `file` is dictionary name (default: %s) \n", g_defaultDictName);
     DISPLAY( "--maxdict=# : limit dictionary to specified size (default: %u) \n", g_defaultMaxDictSize);
@@ -282,10 +283,15 @@ static unsigned parseCoverParameters(const char* stringPtr, ZDICT_cover_params_t
         if (longCommandWArg(&stringPtr, "k=")) { params->k = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
         if (longCommandWArg(&stringPtr, "d=")) { params->d = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
         if (longCommandWArg(&stringPtr, "steps=")) { params->steps = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
+        if (longCommandWArg(&stringPtr, "split=")) {
+          unsigned splitPercentage = readU32FromChar(&stringPtr);
+          params->splitPoint = (double)splitPercentage / 100.0;
+          if (stringPtr[0]==',') { stringPtr++; continue; } else break;
+        }
         return 0;
     }
     if (stringPtr[0] != 0) return 0;
-    DISPLAYLEVEL(4, "cover: k=%u\nd=%u\nsteps=%u\n", params->k, params->d, params->steps);
+    DISPLAYLEVEL(4, "cover: k=%u\nd=%u\nsteps=%u\nsplit=%u\n", params->k, params->d, params->steps, (unsigned)(params->splitPoint * 100));
     return 1;
 }
 
@@ -310,6 +316,7 @@ static ZDICT_cover_params_t defaultCoverParams(void)
     memset(&params, 0, sizeof(params));
     params.d = 8;
     params.steps = 4;
+    params.splitPoint = DEFAULT_SPLITPOINT;
     return params;
 }
 #endif
@@ -398,6 +405,8 @@ int main(int argCount, const char* argv[])
         setRealTimePrio = 0,
         singleThread = 0,
         ultra=0;
+    double compressibility = 0.5;
+    BMK_advancedParams_t adv = BMK_initAdvancedParams();
     unsigned bench_nbSeconds = 3;   /* would be better if this value was synchronized from bench */
     size_t blockSize = 0;
     zstd_operation_mode operation = zom_compress;
@@ -441,7 +450,7 @@ int main(int argCount, const char* argv[])
 #endif
 
     /* preset behaviors */
-    if (exeNameMatch(programName, ZSTD_ZSTDMT)) nbWorkers=0;
+    if (exeNameMatch(programName, ZSTD_ZSTDMT)) nbWorkers=0, singleThread=0;
     if (exeNameMatch(programName, ZSTD_UNZSTD)) operation=zom_decompress;
     if (exeNameMatch(programName, ZSTD_CAT)) { operation=zom_decompress; forceStdout=1; FIO_overwriteMode(); outFileName=stdoutmark; g_displayLevel=1; }   /* supports multiple formats */
     if (exeNameMatch(programName, ZSTD_ZCAT)) { operation=zom_decompress; forceStdout=1; FIO_overwriteMode(); outFileName=stdoutmark; g_displayLevel=1; }  /* behave like zcat, also supports multiple formats */
@@ -609,7 +618,7 @@ int main(int argCount, const char* argv[])
                          /* Decoding */
                     case 'd':
 #ifndef ZSTD_NOBENCH
-                            BMK_setDecodeOnlyMode(1);
+                            adv.mode = BMK_decodeOnly;
                             if (operation==zom_bench) { argument++; break; }  /* benchmark decode (hidden option) */
 #endif
                             operation=zom_decompress; argument++; break;
@@ -702,11 +711,19 @@ int main(int argCount, const char* argv[])
                     case 'p': argument++;
 #ifndef ZSTD_NOBENCH
                         if ((*argument>='0') && (*argument<='9')) {
-                            BMK_setAdditionalParam(readU32FromChar(&argument));
+                            adv.additionalParam = (int)readU32FromChar(&argument);
                         } else
 #endif
                             main_pause=1;
                         break;
+
+                        /* Select compressibility of synthetic sample */
+                    case 'P': 
+                    {   argument++;
+                        compressibility = (double)readU32FromChar(&argument) / 100;
+                    }
+                    break;
+
                         /* unknown command */
                     default : CLEAN_RETURN(badusage(programName));
                     }
@@ -764,7 +781,7 @@ int main(int argCount, const char* argv[])
         DISPLAYLEVEL(3, "Note: %d physical core(s) detected \n", nbWorkers);
     }
 #else
-    (void)singleThread;
+    (void)singleThread; (void)nbWorkers;
 #endif
 
 #ifdef UTIL_HAS_CREATEFILELIST
@@ -807,21 +824,46 @@ int main(int argCount, const char* argv[])
     /* Check if benchmark is selected */
     if (operation==zom_bench) {
 #ifndef ZSTD_NOBENCH
-        BMK_setSeparateFiles(separateFiles);
+        adv.blockSize = blockSize;
-        BMK_setBlockSize(blockSize);
+        adv.nbWorkers = nbWorkers;
-        BMK_setNbWorkers(nbWorkers);
+        adv.realTime = setRealTimePrio;
-        BMK_setRealTime(setRealTimePrio);
+        adv.nbSeconds = bench_nbSeconds;
-        BMK_setNbSeconds(bench_nbSeconds);
+        adv.ldmFlag = ldmFlag;
-        BMK_setLdmFlag(ldmFlag);
+        adv.ldmMinMatch = g_ldmMinMatch;
-        BMK_setLdmMinMatch(g_ldmMinMatch);
+        adv.ldmHashLog = g_ldmHashLog;
-        BMK_setLdmHashLog(g_ldmHashLog);
         if (g_ldmBucketSizeLog != LDM_PARAM_DEFAULT) {
-            BMK_setLdmBucketSizeLog(g_ldmBucketSizeLog);
+            adv.ldmBucketSizeLog = g_ldmBucketSizeLog;
         }
         if (g_ldmHashEveryLog != LDM_PARAM_DEFAULT) {
-            BMK_setLdmHashEveryLog(g_ldmHashEveryLog);
+            adv.ldmHashEveryLog = g_ldmHashEveryLog;
         }
-        BMK_benchFiles(filenameTable, filenameIdx, dictFileName, cLevel, cLevelLast, &compressionParams, g_displayLevel);
+
+        if (cLevel > ZSTD_maxCLevel()) cLevel = ZSTD_maxCLevel();
+        if (cLevelLast > ZSTD_maxCLevel()) cLevelLast = ZSTD_maxCLevel();
+        if (cLevelLast < cLevel) cLevelLast = cLevel;
+        if (cLevelLast > cLevel) 
+            DISPLAYLEVEL(2, "Benchmarking levels from %d to %d\n", cLevel, cLevelLast);
+        if(filenameIdx) {
+            if(separateFiles) {
+                unsigned i;
+                for(i = 0; i < filenameIdx; i++) {
+                    int c;
+                    DISPLAYLEVEL(2, "Benchmarking %s \n", filenameTable[i]);
+                    for(c = cLevel; c <= cLevelLast; c++) {
+                        BMK_benchFilesAdvanced(&filenameTable[i], 1, dictFileName, c, &compressionParams, g_displayLevel, &adv);
+                    }
+                }
+            } else {
+                for(; cLevel <= cLevelLast; cLevel++) {
+                    BMK_benchFilesAdvanced(filenameTable, filenameIdx, dictFileName, cLevel, &compressionParams, g_displayLevel, &adv);
+                }            
+            }
+        } else {
+            for(; cLevel <= cLevelLast; cLevel++) {
+                BMK_syntheticTest(cLevel, compressibility, &compressionParams, g_displayLevel, &adv);
+            }
+        }
+
 #else
         (void)bench_nbSeconds; (void)blockSize; (void)setRealTimePrio; (void)separateFiles;
 #endif

tests/Makefile

@@ -88,13 +88,8 @@ allnothread: fullbench fuzzer paramgrill datagen decodecorpus
 
 dll: fuzzer-dll zstreamtest-dll
 
-zstd:
+PHONY: zstd zstd32 zstd-nolegacy  # must be phony, only external makefile knows how to build them, or if they need an update
-	$(MAKE) -C $(PRGDIR) $@ MOREFLAGS+="$(DEBUGFLAGS)"
+zstd zstd32 zstd-nolegacy:
-
-zstd32:
-	$(MAKE) -C $(PRGDIR) $@ MOREFLAGS+="$(DEBUGFLAGS)"
-
-zstd-nolegacy:
 	$(MAKE) -C $(PRGDIR) $@ MOREFLAGS+="$(DEBUGFLAGS)"
 
 gzstd:
@@ -133,7 +128,7 @@ fullbench fullbench32 : CPPFLAGS += $(MULTITHREAD_CPP)
 fullbench fullbench32 : LDFLAGS += $(MULTITHREAD_LD)
 fullbench fullbench32 : DEBUGFLAGS =   # turn off assert() for speed measurements
 fullbench fullbench32 : $(ZSTD_FILES)
-fullbench fullbench32 : $(PRGDIR)/datagen.c fullbench.c
+fullbench fullbench32 : $(PRGDIR)/datagen.c $(PRGDIR)/bench.c fullbench.c
 	$(CC) $(FLAGS) $^ -o $@$(EXT)
 
 fullbench-lib : zstd-staticLib
@@ -245,6 +240,7 @@ checkTag: checkTag.c $(ZSTDDIR)/zstd.h
 
 clean:
 	$(MAKE) -C $(ZSTDDIR) clean
+	$(MAKE) -C $(PRGDIR) clean
 	@$(RM) -fR $(TESTARTEFACT)
 	@$(RM) -f core *.o tmp* result* *.gcda dictionary *.zst \
         $(PRGDIR)/zstd$(EXT) $(PRGDIR)/zstd32$(EXT) \
@@ -301,11 +297,6 @@ endif
 list:
 	@$(MAKE) -pRrq -f $(lastword $(MAKEFILE_LIST)) : 2>/dev/null | awk -v RS= -F: '/^# File/,/^# Finished Make data base/ {if ($$1 !~ "^[#.]") {print $$1}}' | sort | egrep -v -e '^[^[:alnum:]]' -e '^$@$$' | xargs
 
-.PHONY: zstd-playTests
-zstd-playTests: datagen
-	file $(ZSTD)
-	ZSTD="$(QEMU_SYS) $(ZSTD)" ./playTests.sh $(ZSTDRTTEST)
-
 .PHONY: shortest
 shortest: ZSTDRTTEST=
 shortest: test-zstd
@@ -323,14 +314,21 @@ test32: test-zstd32 test-fullbench32 test-fuzzer32 test-zstream32
 
 test-all: test test32 valgrindTest test-decodecorpus-cli
 
+
+.PHONY: test-zstd test-zstd32 test-zstd-nolegacy
 test-zstd: ZSTD = $(PRGDIR)/zstd
-test-zstd: zstd zstd-playTests
+test-zstd: zstd
 
 test-zstd32: ZSTD = $(PRGDIR)/zstd32
-test-zstd32: zstd32 zstd-playTests
+test-zstd32: zstd32
 
 test-zstd-nolegacy: ZSTD = $(PRGDIR)/zstd-nolegacy
-test-zstd-nolegacy: zstd-nolegacy zstd-playTests
+test-zstd-nolegacy: zstd-nolegacy
+
+test-zstd test-zstd32 test-zstd-nolegacy: datagen
+	file $(ZSTD)
+	ZSTD="$(QEMU_SYS) $(ZSTD)" ./playTests.sh $(ZSTDRTTEST)
+
 
 test-gzstd: gzstd
 	$(PRGDIR)/zstd -f README.md test-zstd-speed.py

tests/fullbench.c

@@ -30,6 +30,7 @@
 #include "zstd.h"        /* ZSTD_versionString */
 #include "util.h"        /* time functions */
 #include "datagen.h"
+#include "bench.h"       /* CustomBench*/
 
 
 /*_************************************
@@ -45,6 +46,8 @@
 #define KNUTH      2654435761U
 #define MAX_MEM    (1984 MB)
 
+#define DEFAULT_CLEVEL 1
+
 #define COMPRESSIBILITY_DEFAULT 0.50
 static const size_t g_sampleSize = 10000000;
 
@@ -89,18 +92,63 @@ static size_t BMK_findMaxMem(U64 requiredMem)
     return (size_t) requiredMem;
 }
 
+/*_*******************************************************
+*  Argument Parsing
+*********************************************************/
+
+#define ERROR_OUT(msg) { DISPLAY("%s \n", msg); exit(1); }
+
+ static unsigned readU32FromChar(const char** stringPtr)
+{
+    const char errorMsg[] = "error: numeric value too large";
+    unsigned result = 0;
+    while ((**stringPtr >='0') && (**stringPtr <='9')) {
+        unsigned const max = (((unsigned)(-1)) / 10) - 1;
+        if (result > max) ERROR_OUT(errorMsg);
+        result *= 10, result += **stringPtr - '0', (*stringPtr)++ ;
+    }
+    if ((**stringPtr=='K') || (**stringPtr=='M')) {
+        unsigned const maxK = ((unsigned)(-1)) >> 10;
+        if (result > maxK) ERROR_OUT(errorMsg);
+        result <<= 10;
+        if (**stringPtr=='M') {
+            if (result > maxK) ERROR_OUT(errorMsg);
+            result <<= 10;
+        }
+        (*stringPtr)++;  /* skip `K` or `M` */
+        if (**stringPtr=='i') (*stringPtr)++;
+        if (**stringPtr=='B') (*stringPtr)++;
+    }
+    return result;
+}
+
+static unsigned longCommandWArg(const char** stringPtr, const char* longCommand)
+{
+    size_t const comSize = strlen(longCommand);
+    int const result = !strncmp(*stringPtr, longCommand, comSize);
+    if (result) *stringPtr += comSize;
+    return result;
+}
 
 /*_*******************************************************
 *  Benchmark wrappers
 *********************************************************/
-size_t local_ZSTD_compress(void* dst, size_t dstSize, void* buff2, const void* src, size_t srcSize)
+
+
+static ZSTD_CCtx* g_zcc = NULL;
+
+size_t local_ZSTD_compress(const void* src, size_t srcSize, void* dst, size_t dstSize, void* buff2)
 {
-    (void)buff2;
+    ZSTD_parameters p;
-    return ZSTD_compress(dst, dstSize, src, srcSize, 1);
+    ZSTD_frameParameters f = {1 /* contentSizeHeader*/, 0, 0};
+    p.fParams = f;
+    p.cParams = *(ZSTD_compressionParameters*)buff2;
+    return ZSTD_compress_advanced (g_zcc,dst, dstSize, src, srcSize, NULL ,0, p);
+    //return ZSTD_compress(dst, dstSize, src, srcSize, cLevel);
 }
 
 static size_t g_cSize = 0;
-size_t local_ZSTD_decompress(void* dst, size_t dstSize, void* buff2, const void* src, size_t srcSize)
+size_t local_ZSTD_decompress(const void* src, size_t srcSize, void* dst, size_t dstSize, void* buff2)
 {
     (void)src; (void)srcSize;
     return ZSTD_decompress(dst, dstSize, buff2, g_cSize);
@@ -110,14 +158,14 @@ static ZSTD_DCtx* g_zdc = NULL;
 
 #ifndef ZSTD_DLL_IMPORT
 extern size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* ctx, const void* src, size_t srcSize);
-size_t local_ZSTD_decodeLiteralsBlock(void* dst, size_t dstSize, void* buff2, const void* src, size_t srcSize)
+size_t local_ZSTD_decodeLiteralsBlock(const void* src, size_t srcSize, void* dst, size_t dstSize, void* buff2)
 {
     (void)src; (void)srcSize; (void)dst; (void)dstSize;
     return ZSTD_decodeLiteralsBlock((ZSTD_DCtx*)g_zdc, buff2, g_cSize);
 }
 
 extern size_t ZSTD_decodeSeqHeaders(ZSTD_DCtx* dctx, int* nbSeq, const void* src, size_t srcSize);
-size_t local_ZSTD_decodeSeqHeaders(void* dst, size_t dstSize, void* buff2, const void* src, size_t srcSize)
+size_t local_ZSTD_decodeSeqHeaders(const void* src, size_t srcSize, void* dst, size_t dstSize, void* buff2)
 {
     int nbSeq;
     (void)src; (void)srcSize; (void)dst; (void)dstSize;
@@ -126,12 +174,15 @@ size_t local_ZSTD_decodeSeqHeaders(void* dst, size_t dstSize, void* buff2, const
 #endif
 
 static ZSTD_CStream* g_cstream= NULL;
-size_t local_ZSTD_compressStream(void* dst, size_t dstCapacity, void* buff2, const void* src, size_t srcSize)
+size_t local_ZSTD_compressStream(const void* src, size_t srcSize, void* dst, size_t dstCapacity, void* buff2)
 {
     ZSTD_outBuffer buffOut;
     ZSTD_inBuffer buffIn;
-    (void)buff2;
+    ZSTD_parameters p;
-    ZSTD_initCStream(g_cstream, 1);
+    ZSTD_frameParameters f = {1 /* contentSizeHeader*/, 0, 0};
+    p.fParams = f;
+    p.cParams = *(ZSTD_compressionParameters*)buff2;
+    ZSTD_initCStream_advanced(g_cstream, NULL, 0, p, ZSTD_CONTENTSIZE_UNKNOWN);
     buffOut.dst = dst;
     buffOut.size = dstCapacity;
     buffOut.pos = 0;
@@ -143,12 +194,11 @@ size_t local_ZSTD_compressStream(void* dst, size_t dstCapacity, void* buff2, con
     return buffOut.pos;
 }
 
-static size_t local_ZSTD_compress_generic_end(void* dst, size_t dstCapacity, void* buff2, const void* src, size_t srcSize)
+static size_t local_ZSTD_compress_generic_end(const void* src, size_t srcSize, void* dst, size_t dstCapacity, void* buff2)
 {
     ZSTD_outBuffer buffOut;
     ZSTD_inBuffer buffIn;
     (void)buff2;
-    ZSTD_CCtx_setParameter(g_cstream, ZSTD_p_compressionLevel, 1);
     buffOut.dst = dst;
     buffOut.size = dstCapacity;
     buffOut.pos = 0;
@@ -159,12 +209,11 @@ static size_t local_ZSTD_compress_generic_end(void* dst, size_t dstCapacity, voi
     return buffOut.pos;
 }
 
-static size_t local_ZSTD_compress_generic_continue(void* dst, size_t dstCapacity, void* buff2, const void* src, size_t srcSize)
+static size_t local_ZSTD_compress_generic_continue(const void* src, size_t srcSize, void* dst, size_t dstCapacity, void* buff2)
 {
     ZSTD_outBuffer buffOut;
     ZSTD_inBuffer buffIn;
     (void)buff2;
-    ZSTD_CCtx_setParameter(g_cstream, ZSTD_p_compressionLevel, 1);
     buffOut.dst = dst;
     buffOut.size = dstCapacity;
     buffOut.pos = 0;
@@ -176,12 +225,11 @@ static size_t local_ZSTD_compress_generic_continue(void* dst, size_t dstCapacity
     return buffOut.pos;
 }
 
-static size_t local_ZSTD_compress_generic_T2_end(void* dst, size_t dstCapacity, void* buff2, const void* src, size_t srcSize)
+static size_t local_ZSTD_compress_generic_T2_end(const void* src, size_t srcSize, void* dst, size_t dstCapacity, void* buff2)
 {
     ZSTD_outBuffer buffOut;
     ZSTD_inBuffer buffIn;
     (void)buff2;
-    ZSTD_CCtx_setParameter(g_cstream, ZSTD_p_compressionLevel, 1);
     ZSTD_CCtx_setParameter(g_cstream, ZSTD_p_nbWorkers, 2);
     buffOut.dst = dst;
     buffOut.size = dstCapacity;
@@ -193,12 +241,11 @@ static size_t local_ZSTD_compress_generic_T2_end(void* dst, size_t dstCapacity,
     return buffOut.pos;
 }
 
-static size_t local_ZSTD_compress_generic_T2_continue(void* dst, size_t dstCapacity, void* buff2, const void* src, size_t srcSize)
+static size_t local_ZSTD_compress_generic_T2_continue(const void* src, size_t srcSize, void* dst, size_t dstCapacity, void* buff2)
 {
     ZSTD_outBuffer buffOut;
     ZSTD_inBuffer buffIn;
     (void)buff2;
-    ZSTD_CCtx_setParameter(g_cstream, ZSTD_p_compressionLevel, 1);
     ZSTD_CCtx_setParameter(g_cstream, ZSTD_p_nbWorkers, 2);
     buffOut.dst = dst;
     buffOut.size = dstCapacity;
@@ -212,7 +259,7 @@ static size_t local_ZSTD_compress_generic_T2_continue(void* dst, size_t dstCapac
 }
 
 static ZSTD_DStream* g_dstream= NULL;
-static size_t local_ZSTD_decompressStream(void* dst, size_t dstCapacity, void* buff2, const void* src, size_t srcSize)
+static size_t local_ZSTD_decompressStream(const void* src, size_t srcSize, void* dst, size_t dstCapacity, void* buff2)
 {
     ZSTD_outBuffer buffOut;
     ZSTD_inBuffer buffIn;
@@ -228,24 +275,28 @@ static size_t local_ZSTD_decompressStream(void* dst, size_t dstCapacity, void* b
     return buffOut.pos;
 }
 
-static ZSTD_CCtx* g_zcc = NULL;
-
 #ifndef ZSTD_DLL_IMPORT
-size_t local_ZSTD_compressContinue(void* dst, size_t dstCapacity, void* buff2, const void* src, size_t srcSize)
+size_t local_ZSTD_compressContinue(const void* src, size_t srcSize, void* dst, size_t dstCapacity, void* buff2)
 {
-    (void)buff2;
+    ZSTD_parameters p;
-    ZSTD_compressBegin(g_zcc, 1 /* compressionLevel */);
+    ZSTD_frameParameters f = {1 /* contentSizeHeader*/, 0, 0};
+    p.fParams = f;
+    p.cParams = *(ZSTD_compressionParameters*)buff2;
+    ZSTD_compressBegin_advanced(g_zcc, NULL, 0, p, srcSize);
     return ZSTD_compressEnd(g_zcc, dst, dstCapacity, src, srcSize);
 }
 
 #define FIRST_BLOCK_SIZE 8
-size_t local_ZSTD_compressContinue_extDict(void* dst, size_t dstCapacity, void* buff2, const void* src, size_t srcSize)
+size_t local_ZSTD_compressContinue_extDict(const void* src, size_t srcSize, void* dst, size_t dstCapacity, void* buff2)
 {
     BYTE firstBlockBuf[FIRST_BLOCK_SIZE];
     
-    (void)buff2;
+    ZSTD_parameters p;
+    ZSTD_frameParameters f = {1 , 0, 0};
+    p.fParams = f;
+    p.cParams = *(ZSTD_compressionParameters*)buff2;
+    ZSTD_compressBegin_advanced(g_zcc, NULL, 0, p, srcSize);
     memcpy(firstBlockBuf, src, FIRST_BLOCK_SIZE);
-    ZSTD_compressBegin(g_zcc, 1);
 
     {   size_t const compressResult = ZSTD_compressContinue(g_zcc, dst, dstCapacity, firstBlockBuf, FIRST_BLOCK_SIZE);
         if (ZSTD_isError(compressResult)) { DISPLAY("local_ZSTD_compressContinue_extDict error : %s\n", ZSTD_getErrorName(compressResult)); return compressResult; }
@@ -255,7 +306,7 @@ size_t local_ZSTD_compressContinue_extDict(void* dst, size_t dstCapacity, void*
     return ZSTD_compressEnd(g_zcc, dst, dstCapacity, (const BYTE*)src + FIRST_BLOCK_SIZE, srcSize - FIRST_BLOCK_SIZE);
 }
 
-size_t local_ZSTD_decompressContinue(void* dst, size_t dstCapacity, void* buff2, const void* src, size_t srcSize)
+size_t local_ZSTD_decompressContinue(const void* src, size_t srcSize, void* dst, size_t dstCapacity, void* buff2)
 {
     size_t regeneratedSize = 0;
     const BYTE* ip = (const BYTE*)buff2;
@@ -282,27 +333,28 @@ size_t local_ZSTD_decompressContinue(void* dst, size_t dstCapacity, void* buff2,
 /*_*******************************************************
 *  Bench functions
 *********************************************************/
-static size_t benchMem(const void* src, size_t srcSize, U32 benchNb)
+static size_t benchMem(const void* src, size_t srcSize, U32 benchNb, int cLevel, ZSTD_compressionParameters* cparams)
 {
     BYTE*  dstBuff;
     size_t const dstBuffSize = ZSTD_compressBound(srcSize);
-    void*  buff2;
+    void*  buff2, *buff1;
     const char* benchName;
-    size_t (*benchFunction)(void* dst, size_t dstSize, void* verifBuff, const void* src, size_t srcSize);
+    BMK_benchFn_t benchFunction;
-    double bestTime = 100000000.;
+    BMK_customReturn_t r;
+    int errorcode = 0;
 
     /* Selection */
     switch(benchNb)
     {
     case 1:
-        benchFunction = local_ZSTD_compress; benchName = "compress(1)";
+        benchFunction = local_ZSTD_compress; benchName = "compress";
         break;
     case 2:
         benchFunction = local_ZSTD_decompress; benchName = "decompress";
         break;
 #ifndef ZSTD_DLL_IMPORT
     case 11:
-        benchFunction = local_ZSTD_compressContinue; benchName = "compressContinue(1)";
+        benchFunction = local_ZSTD_compressContinue; benchName = "compressContinue";
         break;
     case 12:
         benchFunction = local_ZSTD_compressContinue_extDict; benchName = "compressContinue_extDict";
@@ -318,7 +370,7 @@ static size_t benchMem(const void* src, size_t srcSize, U32 benchNb)
         break;
 #endif
     case 41:
-        benchFunction = local_ZSTD_compressStream; benchName = "compressStream(1)";
+        benchFunction = local_ZSTD_compressStream; benchName = "compressStream";
         break;
     case 42:
         benchFunction = local_ZSTD_decompressStream; benchName = "decompressStream";
@@ -347,26 +399,59 @@ static size_t benchMem(const void* src, size_t srcSize, U32 benchNb)
         free(dstBuff); free(buff2);
         return 12;
     }
+    buff1 = buff2;
     if (g_zcc==NULL) g_zcc = ZSTD_createCCtx();
     if (g_zdc==NULL) g_zdc = ZSTD_createDCtx();
     if (g_cstream==NULL) g_cstream = ZSTD_createCStream();
     if (g_dstream==NULL) g_dstream = ZSTD_createDStream();
 
+    /* DISPLAY("params: cLevel %d, wlog %d hlog %d clog %d slog %d slen %d tlen %d strat %d \n"
+        , cLevel, cparams->windowLog, cparams->hashLog, cparams->chainLog, cparams->searchLog, 
+          cparams->searchLength, cparams->targetLength, cparams->strategy);*/
+
+    ZSTD_CCtx_setParameter(g_zcc, ZSTD_p_compressionLevel, cLevel);
+    ZSTD_CCtx_setParameter(g_zcc, ZSTD_p_windowLog, cparams->windowLog);
+    ZSTD_CCtx_setParameter(g_zcc, ZSTD_p_hashLog, cparams->hashLog);
+    ZSTD_CCtx_setParameter(g_zcc, ZSTD_p_chainLog, cparams->chainLog);
+    ZSTD_CCtx_setParameter(g_zcc, ZSTD_p_searchLog, cparams->searchLog);
+    ZSTD_CCtx_setParameter(g_zcc, ZSTD_p_minMatch, cparams->searchLength);
+    ZSTD_CCtx_setParameter(g_zcc, ZSTD_p_targetLength, cparams->targetLength);
+    ZSTD_CCtx_setParameter(g_zcc, ZSTD_p_compressionStrategy, cparams->strategy);
+
+
+    ZSTD_CCtx_setParameter(g_cstream, ZSTD_p_compressionLevel, cLevel);
+    ZSTD_CCtx_setParameter(g_cstream, ZSTD_p_windowLog, cparams->windowLog);
+    ZSTD_CCtx_setParameter(g_cstream, ZSTD_p_hashLog, cparams->hashLog);
+    ZSTD_CCtx_setParameter(g_cstream, ZSTD_p_chainLog, cparams->chainLog);
+    ZSTD_CCtx_setParameter(g_cstream, ZSTD_p_searchLog, cparams->searchLog);
+    ZSTD_CCtx_setParameter(g_cstream, ZSTD_p_minMatch, cparams->searchLength);
+    ZSTD_CCtx_setParameter(g_cstream, ZSTD_p_targetLength, cparams->targetLength);
+    ZSTD_CCtx_setParameter(g_cstream, ZSTD_p_compressionStrategy, cparams->strategy);
+
     /* Preparation */
     switch(benchNb)
     {
+    case 1:
+        buff2 = (void*)cparams;
+        break;
     case 2:
-        g_cSize = ZSTD_compress(buff2, dstBuffSize, src, srcSize, 1);
+        g_cSize = ZSTD_compress(buff2, dstBuffSize, src, srcSize, cLevel);
         break;
 #ifndef ZSTD_DLL_IMPORT
+    case 11:
+        buff2 = (void*)cparams;
+        break;
+    case 12:
+        buff2 = (void*)cparams;
+        break;
     case 13 :
-        g_cSize = ZSTD_compress(buff2, dstBuffSize, src, srcSize, 1);
+        g_cSize = ZSTD_compress(buff2, dstBuffSize, src, srcSize, cLevel);
         break;
     case 31:  /* ZSTD_decodeLiteralsBlock */
         {   blockProperties_t bp;
             ZSTD_frameHeader zfp;
             size_t frameHeaderSize, skippedSize;
-            g_cSize = ZSTD_compress(dstBuff, dstBuffSize, src, srcSize, 1);
+            g_cSize = ZSTD_compress(dstBuff, dstBuffSize, src, srcSize, cLevel);
             frameHeaderSize = ZSTD_getFrameHeader(&zfp, dstBuff, ZSTD_frameHeaderSize_min);
             if (frameHeaderSize==0) frameHeaderSize = ZSTD_frameHeaderSize_min;
             ZSTD_getcBlockSize(dstBuff+frameHeaderSize, dstBuffSize, &bp);  /* Get 1st block type */
@@ -386,8 +471,8 @@ static size_t benchMem(const void* src, size_t srcSize, U32 benchNb)
             const BYTE* ip = dstBuff;
             const BYTE* iend;
             size_t frameHeaderSize, cBlockSize;
-            ZSTD_compress(dstBuff, dstBuffSize, src, srcSize, 1);   /* it would be better to use direct block compression here */
+            ZSTD_compress(dstBuff, dstBuffSize, src, srcSize, cLevel);   /* it would be better to use direct block compression here */
-            g_cSize = ZSTD_compress(dstBuff, dstBuffSize, src, srcSize, 1);
+            g_cSize = ZSTD_compress(dstBuff, dstBuffSize, src, srcSize, cLevel);
             frameHeaderSize = ZSTD_getFrameHeader(&zfp, dstBuff, ZSTD_frameHeaderSize_min);
             if (frameHeaderSize==0) frameHeaderSize = ZSTD_frameHeaderSize_min;
             ip += frameHeaderSize;   /* Skip frame Header */
@@ -409,8 +494,11 @@ static size_t benchMem(const void* src, size_t srcSize, U32 benchNb)
     case 31:
         goto _cleanOut;
 #endif
+    case 41 : 
+        buff2 = (void*)cparams;
+        break;
     case 42 :
-        g_cSize = ZSTD_compress(buff2, dstBuffSize, src, srcSize, 1);
+        g_cSize = ZSTD_compress(buff2, dstBuffSize, src, srcSize, cLevel);
         break;
 
     /* test functions */
@@ -419,58 +507,37 @@ static size_t benchMem(const void* src, size_t srcSize, U32 benchNb)
     default : ;
     }
 
+
      /* warming up memory */
     { size_t i; for (i=0; i<dstBuffSize; i++) dstBuff[i]=(BYTE)i; }
 
-    /* benchmark loop */
-    {   U32 loopNb;
-        U32 nbRounds = (U32)((50 MB) / (srcSize+1)) + 1;   /* initial conservative speed estimate */
-#       define TIME_SEC_MICROSEC    (1*1000000ULL) /* 1 second */
-#       define TIME_SEC_NANOSEC     (1*1000000000ULL) /* 1 second */
-        DISPLAY("%2i- %-30.30s : \r", benchNb, benchName);
-        for (loopNb = 1; loopNb <= g_nbIterations; loopNb++) {
-            UTIL_time_t clockStart;
-            size_t benchResult=0;
-            U32 roundNb;
 
-            UTIL_sleepMilli(5);  /* give processor time to other processes */
+    /* benchmark loop */
-            UTIL_waitForNextTick();
+    {
-            clockStart = UTIL_getTime();
+        r = BMK_benchFunction(benchFunction, buff2, 
-            for (roundNb=0; roundNb < nbRounds; roundNb++) {
+            NULL, NULL,  1, &src, &srcSize, 
-                benchResult = benchFunction(dstBuff, dstBuffSize, buff2, src, srcSize);
+            (void * const * const)&dstBuff, &dstBuffSize, g_nbIterations);
-                if (ZSTD_isError(benchResult)) {
+        if(r.error) {
-                    DISPLAY("ERROR ! %s() => %s !! \n", benchName, ZSTD_getErrorName(benchResult));
+            DISPLAY("ERROR %d ! ! \n", r.error);
-                    exit(1);
+            errorcode = r.error;
-            }   }
+            goto _cleanOut;
-            {   U64 const clockSpanNano = UTIL_clockSpanNano(clockStart);
+        }
-                double const averageTime = (double)clockSpanNano / TIME_SEC_NANOSEC / nbRounds;
+
-                if (clockSpanNano > 0) {
+        DISPLAY("%2u#Speed: %f MB/s - Size: %f MB - %s\n", benchNb, (double)srcSize / r.result.nanoSecPerRun * 1000, (double)r.result.sumOfReturn / 1000000, benchName);
-                    if (averageTime < bestTime) bestTime = averageTime;
-                    assert(bestTime > (1./2000000000));
-                    nbRounds = (U32)(1. / bestTime);   /* aim for 1 sec */
-                    DISPLAY("%2i- %-30.30s : %7.1f MB/s  (%9u)\r",
-                            loopNb, benchName,
-                            (double)srcSize / (1 MB) / bestTime,
-                            (U32)benchResult);
-                } else {
-                    assert(nbRounds < 40000000);  /* avoid overflow */
-                    nbRounds *= 100;
     }
-    }   }   }
-    DISPLAY("%2u\n", benchNb);
     
 _cleanOut:
+    free(buff1);
     free(dstBuff);
-    free(buff2);
     ZSTD_freeCCtx(g_zcc); g_zcc=NULL;
     ZSTD_freeDCtx(g_zdc); g_zdc=NULL;
     ZSTD_freeCStream(g_cstream); g_cstream=NULL;
     ZSTD_freeDStream(g_dstream); g_dstream=NULL;
-    return 0;
+    return errorcode;
 }
 
 
-static int benchSample(U32 benchNb)
+static int benchSample(U32 benchNb, int cLevel, ZSTD_compressionParameters* cparams)
 {
     size_t const benchedSize = g_sampleSize;
     const char* name = "Sample 10MiB";
@@ -486,16 +553,16 @@ static int benchSample(U32 benchNb)
     DISPLAY("\r%79s\r", "");
     DISPLAY(" %s : \n", name);
     if (benchNb)
-        benchMem(origBuff, benchedSize, benchNb);
+        benchMem(origBuff, benchedSize, benchNb, cLevel, cparams);
     else
-        for (benchNb=0; benchNb<100; benchNb++) benchMem(origBuff, benchedSize, benchNb);
+        for (benchNb=0; benchNb<100; benchNb++) benchMem(origBuff, benchedSize, benchNb, cLevel, cparams);
 
     free(origBuff);
     return 0;
 }
 
 
-static int benchFiles(const char** fileNamesTable, const int nbFiles, U32 benchNb)
+static int benchFiles(const char** fileNamesTable, const int nbFiles, U32 benchNb, int cLevel, ZSTD_compressionParameters* cparams)
 {
     /* Loop for each file */
     int fileIdx;
@@ -540,9 +607,9 @@ static int benchFiles(const char** fileNamesTable, const int nbFiles, U32 benchN
         DISPLAY("\r%79s\r", "");
         DISPLAY(" %s : \n", inFileName);
         if (benchNb)
-            benchMem(origBuff, benchedSize, benchNb);
+            benchMem(origBuff, benchedSize, benchNb, cLevel, cparams);
         else
-            for (benchNb=0; benchNb<100; benchNb++) benchMem(origBuff, benchedSize, benchNb);
+            for (benchNb=0; benchNb<100; benchNb++) benchMem(origBuff, benchedSize, benchNb, cLevel, cparams);
 
         free(origBuff);
     }
@@ -567,6 +634,8 @@ static int usage_advanced(const char* exename)
     DISPLAY( " -b#    : test only function # \n");
     DISPLAY( " -i#    : iteration loops [1-9](default : %i)\n", NBLOOPS);
     DISPLAY( " -P#    : sample compressibility (default : %.1f%%)\n", COMPRESSIBILITY_DEFAULT * 100);
+    DISPLAY( " -l#    : benchmark functions at that compression level (default : %i)\n", DEFAULT_CLEVEL);
+    DISPLAY( " --zstd : custom parameter selection. Format same as zstdcli \n");
     return 0;
 }
 
@@ -583,6 +652,8 @@ int main(int argc, const char** argv)
     const char* exename = argv[0];
     const char* input_filename = NULL;
     U32 benchNb = 0, main_pause = 0;
+    int cLevel = DEFAULT_CLEVEL;
+    ZSTD_compressionParameters cparams = ZSTD_getCParams(cLevel, 0, 0);
 
     DISPLAY(WELCOME_MESSAGE);
     if (argc<1) return badusage(exename);
@@ -591,11 +662,29 @@ int main(int argc, const char** argv)
         const char* argument = argv[i];
         assert(argument != NULL);
 
-        /* Commands (note : aggregated commands are allowed) */
+        if (longCommandWArg(&argument, "--zstd=")) {
-        if (argument[0]=='-') {
+            for ( ; ;) {
+                if (longCommandWArg(&argument, "windowLog=") || longCommandWArg(&argument, "wlog=")) { cparams.windowLog = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; }
+                if (longCommandWArg(&argument, "chainLog=") || longCommandWArg(&argument, "clog=")) { cparams.chainLog = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; }
+                if (longCommandWArg(&argument, "hashLog=") || longCommandWArg(&argument, "hlog=")) { cparams.hashLog = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; }
+                if (longCommandWArg(&argument, "searchLog=") || longCommandWArg(&argument, "slog=")) { cparams.searchLog = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; }
+                if (longCommandWArg(&argument, "searchLength=") || longCommandWArg(&argument, "slen=")) { cparams.searchLength = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; }
+                if (longCommandWArg(&argument, "targetLength=") || longCommandWArg(&argument, "tlen=")) { cparams.targetLength = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; }
+                if (longCommandWArg(&argument, "strategy=") || longCommandWArg(&argument, "strat=")) { cparams.strategy = (ZSTD_strategy)(readU32FromChar(&argument)); if (argument[0]==',') { argument++; continue; } else break; }
+                if (longCommandWArg(&argument, "level=") || longCommandWArg(&argument, "lvl=")) { cLevel = (int)readU32FromChar(&argument); cparams = ZSTD_getCParams(cLevel, 0, 0); if (argument[0]==',') { argument++; continue; } else break; }
+                DISPLAY("invalid compression parameter \n");
+                return 1;
+            }
 
-            while (argument[1]!=0) {
+            if (argument[0] != 0) {
+                DISPLAY("invalid --zstd= format\n");
+                return 1; // check the end of string 
+            } else {
+                continue;
+            }
+        } else if (argument[0]=='-') { /* Commands (note : aggregated commands are allowed) */
             argument++;
+            while (argument[0]!=0) {
 
                 switch(argument[0])
                 {
@@ -608,34 +697,34 @@ int main(int argc, const char** argv)
 
                     /* Select specific algorithm to bench */
                 case 'b':
-                    benchNb = 0;
+                    {
-                    while ((argument[1]>= '0') && (argument[1]<= '9')) {
-                        benchNb *= 10;
-                        benchNb += argument[1] - '0';
                         argument++;
-                    }
+                        benchNb = readU32FromChar(&argument);
                         break;
+                    }
 
                     /* Modify Nb Iterations */
                 case 'i':
-                    if ((argument[1] >='0') && (argument[1] <='9')) {
+                    {
-                        int iters = argument[1] - '0';
-                        BMK_SetNbIterations(iters);
                         argument++;
+                        BMK_SetNbIterations((int)readU32FromChar(&argument));
                     }
                     break;
 
                     /* Select compressibility of synthetic sample */
                 case 'P':
-                    {   U32 proba32 = 0;
+                    {   argument++;
-                        while ((argument[1]>= '0') && (argument[1]<= '9')) {
+                        g_compressibility = (double)readU32FromChar(&argument) / 100.;
-                            proba32 *= 10;
-                            proba32 += argument[1] - '0';
-                            argument++;
-                        }
-                        g_compressibility = (double)proba32 / 100.;
                     }
                     break;
+                case 'l':
+                    {   argument++;
+                        cLevel = readU32FromChar(&argument);
+                        cparams = ZSTD_getCParams(cLevel, 0, 0);
+                    }
+                    break;
+
+
 
                     /* Unknown command */
                 default : return badusage(exename);
@@ -648,10 +737,12 @@ int main(int argc, const char** argv)
         if (!input_filename) { input_filename=argument; filenamesStart=i; continue; }
     }
 
+
+
     if (filenamesStart==0)   /* no input file */
-        result = benchSample(benchNb);
+        result = benchSample(benchNb, cLevel, &cparams);
     else
-        result = benchFiles(argv+filenamesStart, argc-filenamesStart, benchNb);
+        result = benchFiles(argv+filenamesStart, argc-filenamesStart, benchNb, cLevel, &cparams);
 
     if (main_pause) { int unused; printf("press enter...\n"); unused = getchar(); (void)unused; }
 

tests/paramgrill.c

@@ -162,18 +162,15 @@ const char* g_stratName[ZSTD_btultra+1] = {
                 "ZSTD_btlazy2 ", "ZSTD_btopt   ", "ZSTD_btultra "};
 
 /* TODO: support additional parameters (more files, fileSizes) */
-
-//TODO: benchMem dctx can't = NULL in new system
 static size_t
 BMK_benchParam(BMK_result_t* resultPtr,
                const void* srcBuffer, size_t srcSize,
                ZSTD_CCtx* ctx, ZSTD_DCtx* dctx, 
                const ZSTD_compressionParameters cParams) {
 
-
     BMK_return_t res = BMK_benchMem(srcBuffer,srcSize, &srcSize, 1, 0, &cParams, NULL, 0, ctx, dctx, 0, "File");
     *resultPtr = res.result;
-    return res.errorCode;
+    return res.error;
 }
 
 static void BMK_printWinner(FILE* f, U32 cLevel, BMK_result_t result, ZSTD_compressionParameters params, size_t srcSize)

tests/playTests.sh

@@ -48,6 +48,8 @@ fileRoundTripTest() {
     $DIFF -q tmp.md5.1 tmp.md5.2
 }
 
+UNAME=$(uname)
+
 isTerminal=false
 if [ -t 0 ] && [ -t 1 ]
 then
@@ -56,7 +58,10 @@ fi
 
 isWindows=false
 INTOVOID="/dev/null"
-DEVDEVICE="/dev/random"
+case "$UNAME" in
+  GNU) DEVDEVICE="/dev/random" ;;
+  *) DEVDEVICE="/dev/zero" ;;
+esac
 case "$OS" in
   Windows*)
     isWindows=true
@@ -65,7 +70,6 @@ case "$OS" in
     ;;
 esac
 
-UNAME=$(uname)
 case "$UNAME" in
   Darwin) MD5SUM="md5 -r" ;;
   FreeBSD) MD5SUM="gmd5sum" ;;
@@ -408,7 +412,7 @@ $ECHO "\n===>  cover dictionary builder : advanced options "
 TESTFILE=../programs/zstdcli.c
 ./datagen > tmpDict
 $ECHO "- Create first dictionary"
-$ZSTD --train-cover=k=46,d=8 *.c ../programs/*.c -o tmpDict
+$ZSTD --train-cover=k=46,d=8,split=80 *.c ../programs/*.c -o tmpDict
 cp $TESTFILE tmp
 $ZSTD -f tmp -D tmpDict
 $ZSTD -d tmp.zst -D tmpDict -fo result
@@ -422,6 +426,11 @@ cmp tmpDict tmpDict1 && die "dictionaries should have different ID !"
 $ECHO "- Create dictionary with size limit"
 $ZSTD --train-cover=steps=8 *.c ../programs/*.c -o tmpDict2 --maxdict=4K
 rm tmp*
+$ECHO "- Compare size of dictionary from 90% training samples with 80% training samples"
+$ZSTD --train-cover=split=90 -r *.c ../programs/*.c
+$ZSTD --train-cover=split=80 -r *.c ../programs/*.c
+$ECHO "- Create dictionary using all samples for both training and testing"
+$ZSTD --train-cover=split=100 -r *.c ../programs/*.c
 
 $ECHO "\n===>  legacy dictionary builder "
 
@@ -544,16 +553,16 @@ $ZSTD --format=xz -V || LZMAMODE=0
 if [ $LZMAMODE -eq 1 ]; then
     $ECHO "xz support detected"
     XZEXE=1
-    xz -V && lzma -V || XZEXE=0
+    xz -Q -V && lzma -Q -V || XZEXE=0
     if [ $XZEXE -eq 1 ]; then
         $ECHO "Testing zstd xz and lzma support"
         ./datagen > tmp
         $ZSTD --format=lzma -f tmp
         $ZSTD --format=xz -f tmp
-        xz -t -v tmp.xz
+        xz -Q -t -v tmp.xz
-        xz -t -v tmp.lzma
+        xz -Q -t -v tmp.lzma
-        xz -f -k tmp
+        xz -Q -f -k tmp
-        lzma -f -k --lzma1 tmp
+        lzma -Q -f -k --lzma1 tmp
         $ZSTD -d -f -v tmp.xz
         $ZSTD -d -f -v tmp.lzma
         rm tmp*
@@ -565,13 +574,13 @@ if [ $LZMAMODE -eq 1 ]; then
         $ECHO "Testing xz and lzma symlinks"
         ./datagen > tmp
         ./xz tmp
-        xz -d tmp.xz
+        xz -Q -d tmp.xz
         ./lzma tmp
-        lzma -d tmp.lzma
+        lzma -Q -d tmp.lzma
         $ECHO "Testing unxz and unlzma symlinks"
-        xz tmp
+        xz -Q tmp
         ./xz -d tmp.xz
-        lzma tmp
+        lzma -Q tmp
         ./lzma -d tmp.lzma
         rm xz unxz lzma unlzma
         rm tmp*
@@ -731,8 +740,14 @@ $ECHO "\n===>  zstd --list/-l error detection tests "
 ! $ZSTD -lv tmp1*
 ! $ZSTD --list -v tmp2 tmp12.zst
 
-$ECHO "\n===>  zstd --list/-l exits 1 when stdin is piped in"
+$ECHO "\n===>  zstd --list/-l errors when presented with stdin / no files"
-! echo "piped STDIN" | $ZSTD --list
+! $ZSTD -l
+! $ZSTD -l -
+! $ZSTD -l < tmp1.zst
+! $ZSTD -l - < tmp1.zst
+! $ZSTD -l - tmp1.zst
+! $ZSTD -l - tmp1.zst < tmp1.zst
+$ZSTD -l tmp1.zst < tmp1.zst # but doesn't error just because stdin is not a tty
 
 $ECHO "\n===>  zstd --list/-l test with null files "
 ./datagen -g0 > tmp5