ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'travisTest' of github.com:facebook/zstd into travisTest

Browse Source
This commit is contained in:
Yann Collet
2018-08-02 11:52:12 -07:00
parent ab1ebd6578 0840d02ecf
commit 064b8e6785
23 changed files with 1355 additions and 636 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.travis.yml
+5
View File
@@ -10,8 +10,13 @@ addons:
matrix:
include:
# Ubuntu 14.04
<<<<<<< HEAD
- env: Cmd='make gcc6install && CC=gcc-6 make -j all
&& make clean && CC=gcc-6 make clean uasan-test-zstd'
=======
- env: Cmd='make test'
- env: Cmd='make gcc6install && CC=gcc-6 make -j all && make clean && CC=gcc-6 make clean uasan-test-zstd </dev/null' # also test when stdin is not a tty
>>>>>>> 0840d02ecf74eae656e0df0d900000d9b0154cde
- env: Cmd='make gcc6install libc6install && CC=gcc-6 make clean uasan-test-zstd32'
- env: Cmd='make gcc7install && CC=gcc-7 make clean uasan-test-zstd'
- env: Cmd='make clang38install && CC=clang-3.8 make clean msan-test-zstd'
Makefile
+2 -1
View File
@@ -63,8 +63,9 @@ zlibwrapper:
$(MAKE) -C $(ZWRAPDIR) test
.PHONY: test
test: MOREFLAGS += -g -DDEBUGLEVEL=1 -Werror
test:
$(MAKE) -C $(PRGDIR) allVariants MOREFLAGS+="-g -DDEBUGLEVEL=1"
MOREFLAGS="$(MOREFLAGS)" $(MAKE) -C $(PRGDIR) allVariants
$(MAKE) -C $(TESTDIR) $@
.PHONY: shortest
build/VS2010/fullbench-dll/fullbench-dll.vcxproj
+2
View File
@@ -167,11 +167,13 @@
<ItemGroup>
<ClCompile Include="..\..\..\lib\common\xxhash.c" />
<ClCompile Include="..\..\..\programs\datagen.c" />
<ClCompile Include="..\..\..\programs\bench.c" />
<ClCompile Include="..\..\..\tests\fullbench.c" />
</ItemGroup>
<ItemGroup>
<ClInclude Include="..\..\..\lib\zstd.h" />
<ClInclude Include="..\..\..\programs\datagen.h" />
<ClInclude Include="..\..\..\programs\bench.h" />
<ClInclude Include="..\..\..\programs\util.h" />
</ItemGroup>
<ItemGroup>
build/VS2010/fullbench/fullbench.vcxproj
+2
View File
@@ -176,6 +176,7 @@
<ClCompile Include="..\..\..\lib\decompress\huf_decompress.c" />
<ClCompile Include="..\..\..\lib\decompress\zstd_decompress.c" />
<ClCompile Include="..\..\..\programs\datagen.c" />
<ClCompile Include="..\..\..\programs\bench.c" />
<ClCompile Include="..\..\..\tests\fullbench.c" />
</ItemGroup>
<ItemGroup>
@@ -197,6 +198,7 @@
<ClInclude Include="..\..\..\lib\legacy\zstd_legacy.h" />
<ClInclude Include="..\..\..\programs\datagen.h" />
<ClInclude Include="..\..\..\programs\util.h" />
<ClInclude Include="..\..\..\programs\bench.h" />
</ItemGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
<ImportGroup Label="ExtensionTargets">
contrib/meson/meson.build
+2
View File
@@ -18,6 +18,7 @@ libzstd_srcs = [
join_paths(common_dir, 'error_private.c'),
join_paths(common_dir, 'xxhash.c'),
join_paths(compress_dir, 'fse_compress.c'),
join_paths(compress_dir, 'hist.c'),
join_paths(compress_dir, 'huf_compress.c'),
join_paths(compress_dir, 'zstd_compress.c'),
join_paths(compress_dir, 'zstd_fast.c'),
@@ -130,6 +131,7 @@ test('fuzzer', fuzzer)
if target_machine.system() != 'windows'
paramgrill = executable('paramgrill',
datagen_c, join_paths(tests_dir, 'paramgrill.c'),
join_paths(programs_dir, 'bench.c'),
include_directories: test_includes,
link_with: libzstd,
dependencies: libm)
contrib/premake/premake4.lua
+6
View File
@@ -0,0 +1,6 @@
-- Include zstd.lua in your GENie or premake4 file, which exposes a project_zstd function
dofile('zstd.lua')
solution 'example'
configurations { 'Debug', 'Release' }
project_zstd('../../lib/')
contrib/premake/zstd.lua
+80
View File
@@ -0,0 +1,80 @@
-- This GENie/premake file copies the behavior of the Makefile in the lib folder.
-- Basic usage: project_zstd(ZSTD_DIR)
function project_zstd(dir, compression, decompression, deprecated, dictbuilder, legacy)
if compression == nil then compression = true end
if decompression == nil then decompression = true end
if deprecated == nil then deprecated = false end
if dictbuilder == nil then dictbuilder = false end
if legacy == nil then legacy = 0 end
if not compression then
dictbuilder = false
deprecated = false
end
if not decompression then
legacy = 0
deprecated = false
end
project 'zstd'
kind 'StaticLib'
language 'C'
files {
dir .. 'zstd.h',
dir .. 'common/**.c',
dir .. 'common/**.h'
}
if compression then
files {
dir .. 'compress/**.c',
dir .. 'compress/**.h'
}
end
if decompression then
files {
dir .. 'decompress/**.c',
dir .. 'decompress/**.h'
}
end
if dictbuilder then
files {
dir .. 'dictBuilder/**.c',
dir .. 'dictBuilder/**.h'
}
end
if deprecated then
files {
dir .. 'deprecated/**.c',
dir .. 'deprecated/**.h'
}
end
if legacy ~= 0 then
if legacy >= 8 then
files {
dir .. 'legacy/zstd_v0' .. (legacy - 7) .. '.*'
}
end
includedirs {
dir .. 'legacy'
}
end
includedirs {
dir,
dir .. 'common'
}
defines {
'XXH_NAMESPACE=ZSTD_',
'ZSTD_LEGACY_SUPPORT=' .. legacy
}
end
doc/zstd_compression_format.md
+4 -4
View File
@@ -488,20 +488,20 @@ For values spanning several bytes, convention is __little-endian__.
__`Size_Format` for `Raw_Literals_Block` and `RLE_Literals_Block`__ :
`Size_Format` uses 1 _or_ 2 bits.
Its value is : `Size_Format = (Header[0]>>2) & 3`
Its value is : `Size_Format = (Literals_Section_Header[0]>>2) & 3`
- `Size_Format` == 00 or 10 : `Size_Format` uses 1 bit.
`Regenerated_Size` uses 5 bits (0-31).
`Literals_Section_Header` uses 1 byte.
`Regenerated_Size = Header[0]>>3`
`Regenerated_Size = Literals_Section_Header[0]>>3`
- `Size_Format` == 01 : `Size_Format` uses 2 bits.
`Regenerated_Size` uses 12 bits (0-4095).
`Literals_Section_Header` uses 2 bytes.
`Regenerated_Size = (Header[0]>>4) + (Header[1]<<4)`
`Regenerated_Size = (Literals_Section_Header[0]>>4) + (Literals_Section_Header[1]<<4)`
- `Size_Format` == 11 : `Size_Format` uses 2 bits.
`Regenerated_Size` uses 20 bits (0-1048575).
`Literals_Section_Header` uses 3 bytes.
`Regenerated_Size = (Header[0]>>4) + (Header[1]<<4) + (Header[2]<<12)`
`Regenerated_Size = (Literals_Section_Header[0]>>4) + (Literals_Section_Header[1]<<4) + (Literals_Section_Header[2]<<12)`
Only Stream1 is present for these cases.
Note : it's allowed to represent a short value (for example `13`)
lib/common/compiler.h
+13 -8
View File
@@ -88,15 +88,20 @@
#endif
#endif
/* prefetch */
#if defined(_MSC_VER) && (defined(_M_X64) || defined(_M_I86)) /* _mm_prefetch() is not defined outside of x86/x64 */
# include <mmintrin.h> /* https://msdn.microsoft.com/fr-fr/library/84szxsww(v=vs.90).aspx */
# define PREFETCH(ptr) _mm_prefetch((const char*)ptr, _MM_HINT_T0)
#elif defined(__GNUC__) && ( (__GNUC__ >= 4) || ( (__GNUC__ == 3) && (__GNUC_MINOR__ >= 1) ) )
# define PREFETCH(ptr) __builtin_prefetch(ptr, 0, 0)
#else
/* prefetch
* can be disabled, by declaring NO_PREFETCH macro */
#if defined(NO_PREFETCH)
# define PREFETCH(ptr) /* disabled */
#endif
#else
# if defined(_MSC_VER) && (defined(_M_X64) || defined(_M_I86)) /* _mm_prefetch() is not defined outside of x86/x64 */
# include <mmintrin.h> /* https://msdn.microsoft.com/fr-fr/library/84szxsww(v=vs.90).aspx */
# define PREFETCH(ptr) _mm_prefetch((const char*)ptr, _MM_HINT_T0)
# elif defined(__GNUC__) && ( (__GNUC__ >= 4) || ( (__GNUC__ == 3) && (__GNUC_MINOR__ >= 1) ) )
# define PREFETCH(ptr) __builtin_prefetch(ptr, 0, 0)
# else
# define PREFETCH(ptr) /* disabled */
# endif
#endif /* NO_PREFETCH */
/* disable warnings */
#ifdef _MSC_VER /* Visual Studio */
lib/dictBuilder/cover.c
+48 -13
View File
@@ -39,6 +39,7 @@
* Constants
***************************************/
#define COVER_MAX_SAMPLES_SIZE (sizeof(size_t) == 8 ? ((U32)-1) : ((U32)1 GB))
#define DEFAULT_SPLITPOINT 1.0
/*-*************************************
* Console display
@@ -203,6 +204,8 @@ typedef struct {
size_t *offsets;
const size_t *samplesSizes;
size_t nbSamples;
size_t nbTrainSamples;
size_t nbTestSamples;
U32 *suffix;
size_t suffixSize;
U32 *freqs;
@@ -222,7 +225,7 @@ static COVER_ctx_t *g_ctx = NULL;
*/
static size_t COVER_sum(const size_t *samplesSizes, unsigned nbSamples) {
size_t sum = 0;
size_t i;
unsigned i;
for (i = 0; i < nbSamples; ++i) {
sum += samplesSizes[i];
}
@@ -494,6 +497,10 @@ static int COVER_checkParameters(ZDICT_cover_params_t parameters,
if (parameters.d > parameters.k) {
return 0;
}
/* 0 < splitPoint <= 1 */
if (parameters.splitPoint <= 0 || parameters.splitPoint > 1){
return 0;
}
return 1;
}
@@ -531,9 +538,14 @@ static void COVER_ctx_destroy(COVER_ctx_t *ctx) {
*/
static int COVER_ctx_init(COVER_ctx_t *ctx, const void *samplesBuffer,
const size_t *samplesSizes, unsigned nbSamples,
unsigned d) {
unsigned d, double splitPoint) {
const BYTE *const samples = (const BYTE *)samplesBuffer;
const size_t totalSamplesSize = COVER_sum(samplesSizes, nbSamples);
/* Split samples into testing and training sets */
const unsigned nbTrainSamples = splitPoint < 1.0 ? (unsigned)((double)nbSamples * splitPoint) : nbSamples;
const unsigned nbTestSamples = splitPoint < 1.0 ? nbSamples - nbTrainSamples : nbSamples;
const size_t trainingSamplesSize = splitPoint < 1.0 ? COVER_sum(samplesSizes, nbTrainSamples) : totalSamplesSize;
const size_t testSamplesSize = splitPoint < 1.0 ? COVER_sum(samplesSizes + nbTrainSamples, nbTestSamples) : totalSamplesSize;
/* Checks */
if (totalSamplesSize < MAX(d, sizeof(U64)) ||
totalSamplesSize >= (size_t)COVER_MAX_SAMPLES_SIZE) {
@@ -541,15 +553,29 @@ static int COVER_ctx_init(COVER_ctx_t *ctx, const void *samplesBuffer,
(U32)(totalSamplesSize>>20), (COVER_MAX_SAMPLES_SIZE >> 20));
return 0;
}
/* Check if there are at least 5 training samples */
if (nbTrainSamples < 5) {
DISPLAYLEVEL(1, "Total number of training samples is %u and is invalid.", nbTrainSamples);
return 0;
}
/* Check if there's testing sample */
if (nbTestSamples < 1) {
DISPLAYLEVEL(1, "Total number of testing samples is %u and is invalid.", nbTestSamples);
return 0;
}
/* Zero the context */
memset(ctx, 0, sizeof(*ctx));
DISPLAYLEVEL(2, "Training on %u samples of total size %u\n", nbSamples,
(U32)totalSamplesSize);
DISPLAYLEVEL(2, "Training on %u samples of total size %u\n", nbTrainSamples,
(U32)trainingSamplesSize);
DISPLAYLEVEL(2, "Testing on %u samples of total size %u\n", nbTestSamples,
(U32)testSamplesSize);
ctx->samples = samples;
ctx->samplesSizes = samplesSizes;
ctx->nbSamples = nbSamples;
ctx->nbTrainSamples = nbTrainSamples;
ctx->nbTestSamples = nbTestSamples;
/* Partial suffix array */
ctx->suffixSize = totalSamplesSize - MAX(d, sizeof(U64)) + 1;
ctx->suffixSize = trainingSamplesSize - MAX(d, sizeof(U64)) + 1;
ctx->suffix = (U32 *)malloc(ctx->suffixSize * sizeof(U32));
/* Maps index to the dmerID */
ctx->dmerAt = (U32 *)malloc(ctx->suffixSize * sizeof(U32));
@@ -563,7 +589,7 @@ static int COVER_ctx_init(COVER_ctx_t *ctx, const void *samplesBuffer,
ctx->freqs = NULL;
ctx->d = d;
/* Fill offsets from the samlesSizes */
/* Fill offsets from the samplesSizes */
{
U32 i;
ctx->offsets[0] = 0;
@@ -665,7 +691,7 @@ ZDICTLIB_API size_t ZDICT_trainFromBuffer_cover(
BYTE* const dict = (BYTE*)dictBuffer;
COVER_ctx_t ctx;
COVER_map_t activeDmers;
parameters.splitPoint = 1.0;
/* Initialize global data */
g_displayLevel = parameters.zParams.notificationLevel;
/* Checks */
@@ -684,7 +710,7 @@ ZDICTLIB_API size_t ZDICT_trainFromBuffer_cover(
}
/* Initialize context and activeDmers */
if (!COVER_ctx_init(&ctx, samplesBuffer, samplesSizes, nbSamples,
parameters.d)) {
parameters.d, parameters.splitPoint)) {
return ERROR(GENERIC);
}
if (!COVER_map_init(&activeDmers, parameters.k - parameters.d + 1)) {
@@ -839,7 +865,7 @@ typedef struct COVER_tryParameters_data_s {
} COVER_tryParameters_data_t;
/**
* Tries a set of parameters and upates the COVER_best_t with the results.
* Tries a set of parameters and updates the COVER_best_t with the results.
* This function is thread safe if zstd is compiled with multithreaded support.
* It takes its parameters as an *OWNING* opaque pointer to support threading.
*/
@@ -870,7 +896,7 @@ static void COVER_tryParameters(void *opaque) {
dictBufferCapacity, parameters);
dictBufferCapacity = ZDICT_finalizeDictionary(
dict, dictBufferCapacity, dict + tail, dictBufferCapacity - tail,
ctx->samples, ctx->samplesSizes, (unsigned)ctx->nbSamples,
ctx->samples, ctx->samplesSizes, (unsigned)ctx->nbTrainSamples,
parameters.zParams);
if (ZDICT_isError(dictBufferCapacity)) {
DISPLAYLEVEL(1, "Failed to finalize dictionary\n");
@@ -889,7 +915,8 @@ static void COVER_tryParameters(void *opaque) {
/* Allocate dst with enough space to compress the maximum sized sample */
{
size_t maxSampleSize = 0;
for (i = 0; i < ctx->nbSamples; ++i) {
i = parameters.splitPoint < 1.0 ? ctx->nbTrainSamples : 0;
for (; i < ctx->nbSamples; ++i) {
maxSampleSize = MAX(ctx->samplesSizes[i], maxSampleSize);
}
dstCapacity = ZSTD_compressBound(maxSampleSize);
@@ -904,7 +931,8 @@ static void COVER_tryParameters(void *opaque) {
}
/* Compress each sample and sum their sizes (or error) */
totalCompressedSize = dictBufferCapacity;
for (i = 0; i < ctx->nbSamples; ++i) {
i = parameters.splitPoint < 1.0 ? ctx->nbTrainSamples : 0;
for (; i < ctx->nbSamples; ++i) {
const size_t size = ZSTD_compress_usingCDict(
cctx, dst, dstCapacity, ctx->samples + ctx->offsets[i],
ctx->samplesSizes[i], cdict);
@@ -941,6 +969,8 @@ ZDICTLIB_API size_t ZDICT_optimizeTrainFromBuffer_cover(
ZDICT_cover_params_t *parameters) {
/* constants */
const unsigned nbThreads = parameters->nbThreads;
const double splitPoint =
parameters->splitPoint <= 0.0 ? DEFAULT_SPLITPOINT : parameters->splitPoint;
const unsigned kMinD = parameters->d == 0 ? 6 : parameters->d;
const unsigned kMaxD = parameters->d == 0 ? 8 : parameters->d;
const unsigned kMinK = parameters->k == 0 ? 50 : parameters->k;
@@ -958,6 +988,10 @@ ZDICTLIB_API size_t ZDICT_optimizeTrainFromBuffer_cover(
POOL_ctx *pool = NULL;
/* Checks */
if (splitPoint <= 0 || splitPoint > 1) {
LOCALDISPLAYLEVEL(displayLevel, 1, "Incorrect parameters\n");
return ERROR(GENERIC);
}
if (kMinK < kMaxD || kMaxK < kMinK) {
LOCALDISPLAYLEVEL(displayLevel, 1, "Incorrect parameters\n");
return ERROR(GENERIC);
@@ -988,7 +1022,7 @@ ZDICTLIB_API size_t ZDICT_optimizeTrainFromBuffer_cover(
/* Initialize the context for this value of d */
COVER_ctx_t ctx;
LOCALDISPLAYLEVEL(displayLevel, 3, "d=%u\n", d);
if (!COVER_ctx_init(&ctx, samplesBuffer, samplesSizes, nbSamples, d)) {
if (!COVER_ctx_init(&ctx, samplesBuffer, samplesSizes, nbSamples, d, splitPoint)) {
LOCALDISPLAYLEVEL(displayLevel, 1, "Failed to initialize context\n");
COVER_best_destroy(&best);
POOL_free(pool);
@@ -1013,6 +1047,7 @@ ZDICTLIB_API size_t ZDICT_optimizeTrainFromBuffer_cover(
data->parameters = *parameters;
data->parameters.k = k;
data->parameters.d = d;
data->parameters.splitPoint = splitPoint;
data->parameters.steps = kSteps;
data->parameters.zParams.notificationLevel = g_displayLevel;
/* Check the parameters */
lib/dictBuilder/zdict.h
+1
View File
@@ -86,6 +86,7 @@ typedef struct {
unsigned d; /* dmer size : constraint: 0 < d <= k : Reasonable range [6, 16] */
unsigned steps; /* Number of steps : Only used for optimization : 0 means default (32) : Higher means more parameters checked */
unsigned nbThreads; /* Number of threads : constraint: 0 < nbThreads : 1 means single-threaded : Only used for optimization : Ignored if ZSTD_MULTITHREAD is not defined */
double splitPoint; /* Percentage of samples used for training: the first nbSamples * splitPoint samples will be used to training, the last nbSamples * (1 - splitPoint) samples will be used for testing, 0 means default (1.0), 1.0 when all samples are used for both training and testing */
ZDICT_params_t zParams;
} ZDICT_cover_params_t;
programs/README.md
+1 -1
View File
@@ -150,7 +150,7 @@ Advanced arguments :
Dictionary builder :
--train ## : create a dictionary from a training set of files
--train-cover[=k=#,d=#,steps=#] : use the cover algorithm with optional args
--train-cover[=k=#,d=#,steps=#,split=#] : use the cover algorithm with optional args
--train-legacy[=s=#] : use the legacy algorithm with selectivity (default: 9)
-o file : `file` is dictionary name (default: dictionary)
--maxdict=# : limit dictionary to specified size (default: 112640)
programs/bench.c
+686 -418
View File
@@ -42,6 +42,7 @@
#include "datagen.h" /* RDG_genBuffer */
#include "xxhash.h"
#include "bench.h"
#include "zstd_errors.h"
/* *************************************
@@ -64,9 +65,6 @@
static const size_t maxMemory = (sizeof(size_t)==4) ? (2 GB - 64 MB) : (size_t)(1ULL << ((sizeof(size_t)*8)-31));
static U32 g_compressibilityDefault = 50;
/* *************************************
* console display
***************************************/
@@ -90,88 +88,60 @@ static UTIL_time_t g_displayClock = UTIL_TIME_INITIALIZER;
# define DEBUG 0
#endif
#define DEBUGOUTPUT(...) { if (DEBUG) DISPLAY(__VA_ARGS__); }
#define EXM_THROW(error, ...) { \
#define EXM_THROW_INT(errorNum, ...) { \
DEBUGOUTPUT("%s: %i: \n", __FILE__, __LINE__); \
DISPLAYLEVEL(1, "Error %i : ", error); \
DISPLAYLEVEL(1, "Error %i : ", errorNum); \
DISPLAYLEVEL(1, __VA_ARGS__); \
DISPLAYLEVEL(1, " \n"); \
exit(error); \
return errorNum; \
}
#define EXM_THROW(errorNum, retType, ...) { \
retType r; \
memset(&r, 0, sizeof(retType)); \
DEBUGOUTPUT("%s: %i: \n", __FILE__, __LINE__); \
DISPLAYLEVEL(1, "Error %i : ", errorNum); \
DISPLAYLEVEL(1, __VA_ARGS__); \
DISPLAYLEVEL(1, " \n"); \
r.error = errorNum; \
return r; \
}
/* error without displaying */
#define EXM_THROW_ND(errorNum, retType, ...) { \
retType r; \
memset(&r, 0, sizeof(retType)); \
DEBUGOUTPUT("%s: %i: \n", __FILE__, __LINE__); \
DEBUGOUTPUT("Error %i : ", errorNum); \
DEBUGOUTPUT(__VA_ARGS__); \
DEBUGOUTPUT(" \n"); \
r.error = errorNum; \
return r; \
}
/* *************************************
* Benchmark Parameters
***************************************/
static int g_additionalParam = 0;
static U32 g_decodeOnly = 0;
void BMK_setAdditionalParam(int additionalParam) { g_additionalParam=additionalParam; }
//TODO : Deal with DISPLAYLEVEL for all these set functions
static U32 g_nbSeconds = BMK_TIMETEST_DEFAULT_S;
void BMK_setNbSeconds(unsigned nbSeconds)
{
g_nbSeconds = nbSeconds;
DISPLAY("- test >= %u seconds per compression / decompression - \n", g_nbSeconds);
BMK_advancedParams_t BMK_initAdvancedParams(void) {
BMK_advancedParams_t res = {
BMK_both, /* mode */
BMK_timeMode, /* loopMode */
BMK_TIMETEST_DEFAULT_S, /* nbSeconds */
0, /* blockSize */
0, /* nbWorkers */
0, /* realTime */
0, /* additionalParam */
0, /* ldmFlag */
0, /* ldmMinMatch */
0, /* ldmHashLog */
0, /* ldmBuckSizeLog */
0 /* ldmHashEveryLog */
};
return res;
}
static size_t g_blockSize = 0;
void BMK_setBlockSize(size_t blockSize)
{
g_blockSize = blockSize;
if (g_blockSize) DISPLAY("using blocks of size %u KB \n", (U32)(blockSize>>10));
}
void BMK_setDecodeOnlyMode(unsigned decodeFlag) { g_decodeOnly = (decodeFlag>0); }
static U32 g_nbWorkers = 0;
void BMK_setNbWorkers(unsigned nbWorkers) {
#ifndef ZSTD_MULTITHREAD
if (nbWorkers > 0) DISPLAY("Note : multi-threading is disabled \n");
#endif
g_nbWorkers = nbWorkers;
}
static U32 g_realTime = 0;
void BMK_setRealTime(unsigned priority) {
g_realTime = (priority>0);
}
static U32 g_separateFiles = 0;
void BMK_setSeparateFiles(unsigned separate) {
g_separateFiles = (separate>0);
}
static U32 g_ldmFlag = 0;
void BMK_setLdmFlag(unsigned ldmFlag) {
g_ldmFlag = ldmFlag;
}
static U32 g_ldmMinMatch = 0;
void BMK_setLdmMinMatch(unsigned ldmMinMatch) {
g_ldmMinMatch = ldmMinMatch;
}
static U32 g_ldmHashLog = 0;
void BMK_setLdmHashLog(unsigned ldmHashLog) {
g_ldmHashLog = ldmHashLog;
}
#define BMK_LDM_PARAM_NOTSET 9999
static U32 g_ldmBucketSizeLog = BMK_LDM_PARAM_NOTSET;
void BMK_setLdmBucketSizeLog(unsigned ldmBucketSizeLog) {
g_ldmBucketSizeLog = ldmBucketSizeLog;
}
static U32 g_ldmHashEveryLog = BMK_LDM_PARAM_NOTSET;
void BMK_setLdmHashEveryLog(unsigned ldmHashEveryLog) {
g_ldmHashEveryLog = ldmHashEveryLog;
}
/* ********************************************************
* Bench functions
@@ -186,344 +156,638 @@ typedef struct {
size_t resSize;
} blockParam_t;
struct BMK_timeState_t{
unsigned nbLoops;
U64 timeRemaining;
UTIL_time_t coolTime;
U64 fastestTime;
};
#undef MIN
#undef MAX
#define MIN(a,b) ((a) < (b) ? (a) : (b))
#define MAX(a,b) ((a) > (b) ? (a) : (b))
BMK_return_t BMK_benchMem(const void* srcBuffer, size_t srcSize,
const size_t* fileSizes, unsigned nbFiles,
const int cLevel, const ZSTD_compressionParameters* comprParams,
const void* dictBuffer, size_t dictBufferSize,
ZSTD_CCtx* ctx, ZSTD_DCtx* dctx,
int displayLevel, const char* displayName)
static void BMK_initCCtx(ZSTD_CCtx* ctx,
const void* dictBuffer, size_t dictBufferSize, int cLevel,
const ZSTD_compressionParameters* comprParams, const BMK_advancedParams_t* adv) {
if (adv->nbWorkers==1) {
ZSTD_CCtx_setParameter(ctx, ZSTD_p_nbWorkers, 0);
} else {
ZSTD_CCtx_setParameter(ctx, ZSTD_p_nbWorkers, adv->nbWorkers);
}
ZSTD_CCtx_setParameter(ctx, ZSTD_p_compressionLevel, cLevel);
ZSTD_CCtx_setParameter(ctx, ZSTD_p_enableLongDistanceMatching, adv->ldmFlag);
ZSTD_CCtx_setParameter(ctx, ZSTD_p_ldmMinMatch, adv->ldmMinMatch);
ZSTD_CCtx_setParameter(ctx, ZSTD_p_ldmHashLog, adv->ldmHashLog);
ZSTD_CCtx_setParameter(ctx, ZSTD_p_ldmBucketSizeLog, adv->ldmBucketSizeLog);
ZSTD_CCtx_setParameter(ctx, ZSTD_p_ldmHashEveryLog, adv->ldmHashEveryLog);
ZSTD_CCtx_setParameter(ctx, ZSTD_p_windowLog, comprParams->windowLog);
ZSTD_CCtx_setParameter(ctx, ZSTD_p_hashLog, comprParams->hashLog);
ZSTD_CCtx_setParameter(ctx, ZSTD_p_chainLog, comprParams->chainLog);
ZSTD_CCtx_setParameter(ctx, ZSTD_p_searchLog, comprParams->searchLog);
ZSTD_CCtx_setParameter(ctx, ZSTD_p_minMatch, comprParams->searchLength);
ZSTD_CCtx_setParameter(ctx, ZSTD_p_targetLength, comprParams->targetLength);
ZSTD_CCtx_setParameter(ctx, ZSTD_p_compressionStrategy, comprParams->strategy);
ZSTD_CCtx_loadDictionary(ctx, dictBuffer, dictBufferSize);
}
static void BMK_initDCtx(ZSTD_DCtx* dctx,
const void* dictBuffer, size_t dictBufferSize) {
ZSTD_DCtx_loadDictionary(dctx, dictBuffer, dictBufferSize);
}
typedef struct {
ZSTD_CCtx* ctx;
const void* dictBuffer;
size_t dictBufferSize;
int cLevel;
const ZSTD_compressionParameters* comprParams;
const BMK_advancedParams_t* adv;
} BMK_initCCtxArgs;
static size_t local_initCCtx(void* payload) {
BMK_initCCtxArgs* ag = (BMK_initCCtxArgs*)payload;
BMK_initCCtx(ag->ctx, ag->dictBuffer, ag->dictBufferSize, ag->cLevel, ag->comprParams, ag->adv);
return 0;
}
typedef struct {
ZSTD_DCtx* dctx;
const void* dictBuffer;
size_t dictBufferSize;
} BMK_initDCtxArgs;
static size_t local_initDCtx(void* payload) {
BMK_initDCtxArgs* ag = (BMK_initDCtxArgs*)payload;
BMK_initDCtx(ag->dctx, ag->dictBuffer, ag->dictBufferSize);
return 0;
}
/* additional argument is just the context */
static size_t local_defaultCompress(
const void* srcBuffer, size_t srcSize,
void* dstBuffer, size_t dstSize,
void* addArgs) {
size_t moreToFlush = 1;
ZSTD_CCtx* ctx = (ZSTD_CCtx*)addArgs;
ZSTD_inBuffer in;
ZSTD_outBuffer out;
in.src = srcBuffer;
in.size = srcSize;
in.pos = 0;
out.dst = dstBuffer;
out.size = dstSize;
out.pos = 0;
while (moreToFlush) {
if(out.pos == out.size) {
return (size_t)-ZSTD_error_dstSize_tooSmall;
}
moreToFlush = ZSTD_compress_generic(ctx, &out, &in, ZSTD_e_end);
if (ZSTD_isError(moreToFlush)) {
return moreToFlush;
}
}
return out.pos;
}
/* additional argument is just the context */
static size_t local_defaultDecompress(
const void* srcBuffer, size_t srcSize,
void* dstBuffer, size_t dstSize,
void* addArgs) {
size_t moreToFlush = 1;
ZSTD_DCtx* dctx = (ZSTD_DCtx*)addArgs;
ZSTD_inBuffer in;
ZSTD_outBuffer out;
in.src = srcBuffer;
in.size = srcSize;
in.pos = 0;
out.dst = dstBuffer;
out.size = dstSize;
out.pos = 0;
while (moreToFlush) {
if(out.pos == out.size) {
return (size_t)-ZSTD_error_dstSize_tooSmall;
}
moreToFlush = ZSTD_decompress_generic(dctx,
&out, &in);
if (ZSTD_isError(moreToFlush)) {
return moreToFlush;
}
}
return out.pos;
}
/* initFn will be measured once, bench fn will be measured x times */
/* benchFn should return error value or out Size */
/* takes # of blocks and list of size & stuff for each. */
/* only does looping */
/* note time per loop could be zero if interval too short */
BMK_customReturn_t BMK_benchFunction(
BMK_benchFn_t benchFn, void* benchPayload,
BMK_initFn_t initFn, void* initPayload,
size_t blockCount,
const void* const * const srcBlockBuffers, const size_t* srcBlockSizes,
void* const * const dstBlockBuffers, const size_t* dstBlockCapacities,
unsigned nbLoops) {
size_t srcSize = 0, dstSize = 0, ind = 0;
U64 totalTime;
BMK_customReturn_t retval;
UTIL_time_t clockStart;
{
unsigned i;
for(i = 0; i < blockCount; i++) {
memset(dstBlockBuffers[i], 0xE5, dstBlockCapacities[i]); /* warm up and erase result buffer */
}
UTIL_sleepMilli(5); /* give processor time to other processes */
UTIL_waitForNextTick();
}
if(!nbLoops) {
EXM_THROW_ND(1, BMK_customReturn_t, "nbLoops must be nonzero \n");
}
for(ind = 0; ind < blockCount; ind++) {
srcSize += srcBlockSizes[ind];
}
{
unsigned i, j, firstIter = 1;
clockStart = UTIL_getTime();
if(initFn != NULL) { initFn(initPayload); }
for(i = 0; i < nbLoops; i++) {
for(j = 0; j < blockCount; j++) {
size_t res = benchFn(srcBlockBuffers[j], srcBlockSizes[j], dstBlockBuffers[j], dstBlockCapacities[j], benchPayload);
if(ZSTD_isError(res)) {
EXM_THROW_ND(2, BMK_customReturn_t, "Function benchmarking failed on block %u of size %u : %s \n",
j, (U32)dstBlockCapacities[j], ZSTD_getErrorName(res));
} else if(firstIter) {
dstSize += res;
}
}
firstIter = 0;
}
totalTime = UTIL_clockSpanNano(clockStart);
}
retval.error = 0;
retval.result.nanoSecPerRun = totalTime / nbLoops;
retval.result.sumOfReturn = dstSize;
return retval;
}
#define MINUSABLETIME 500000000ULL /* 0.5 seconds in ns */
void BMK_resetTimeState(BMK_timedFnState_t* r, unsigned nbSeconds) {
r->nbLoops = 1;
r->timeRemaining = (U64)nbSeconds * TIMELOOP_NANOSEC;
r->coolTime = UTIL_getTime();
r->fastestTime = (U64)(-1LL);
}
BMK_timedFnState_t* BMK_createTimeState(unsigned nbSeconds) {
BMK_timedFnState_t* r = (BMK_timedFnState_t*)malloc(sizeof(struct BMK_timeState_t));
BMK_resetTimeState(r, nbSeconds);
return r;
}
void BMK_freeTimeState(BMK_timedFnState_t* state) {
free(state);
}
BMK_customTimedReturn_t BMK_benchFunctionTimed(
BMK_timedFnState_t* cont,
BMK_benchFn_t benchFn, void* benchPayload,
BMK_initFn_t initFn, void* initPayload,
size_t blockCount,
const void* const* const srcBlockBuffers, const size_t* srcBlockSizes,
void* const* const dstBlockBuffers, const size_t* dstBlockCapacities)
{
size_t const blockSize = ((g_blockSize>=32 && !g_decodeOnly) ? g_blockSize : srcSize) + (!srcSize) /* avoid div by 0 */ ;
U32 const maxNbBlocks = (U32) ((srcSize + (blockSize-1)) / blockSize) + nbFiles;
blockParam_t* const blockTable = (blockParam_t*) malloc(maxNbBlocks * sizeof(blockParam_t));
size_t const maxCompressedSize = ZSTD_compressBound(srcSize) + (maxNbBlocks * 1024); /* add some room for safety */
void* const compressedBuffer = malloc(maxCompressedSize);
void* resultBuffer = malloc(srcSize);
BMK_return_t results;
U64 fastest = cont->fastestTime;
int completed = 0;
BMK_customTimedReturn_t r;
r.completed = 0;
while(!r.completed && !completed)
{
/* Overheat protection */
if (UTIL_clockSpanMicro(cont->coolTime) > ACTIVEPERIOD_MICROSEC) {
DEBUGOUTPUT("\rcooling down ... \r");
UTIL_sleep(COOLPERIOD_SEC);
cont->coolTime = UTIL_getTime();
}
r.result = BMK_benchFunction(benchFn, benchPayload, initFn, initPayload,
blockCount, srcBlockBuffers, srcBlockSizes, dstBlockBuffers, dstBlockCapacities, cont->nbLoops);
if(r.result.error) { /* completed w/ error */
r.completed = 1;
return r;
}
{ U64 const loopDuration = r.result.result.nanoSecPerRun * cont->nbLoops;
r.completed = (cont->timeRemaining <= loopDuration);
cont->timeRemaining -= loopDuration;
if (loopDuration > 0) {
fastest = MIN(fastest, r.result.result.nanoSecPerRun);
if(loopDuration >= MINUSABLETIME) {
r.result.result.nanoSecPerRun = fastest;
cont->fastestTime = fastest;
}
cont->nbLoops = (U32)(TIMELOOP_NANOSEC / r.result.result.nanoSecPerRun) + 1;
} else {
const unsigned multiplier = 2;
assert(cont->nbLoops < ((unsigned)-1) / multiplier); /* avoid overflow */
cont->nbLoops *= multiplier;
}
if(loopDuration < MINUSABLETIME) { /* don't report results which have time too low */
continue;
}
}
completed = 1;
}
return r;
}
/* benchMem with no allocation */
static BMK_return_t BMK_benchMemAdvancedNoAlloc(
const void ** const srcPtrs, size_t* const srcSizes,
void** const cPtrs, size_t* const cSizes,
void** const resPtrs, size_t* const resSizes,
void* resultBuffer, void* compressedBuffer,
const size_t maxCompressedSize,
BMK_timedFnState_t* timeStateCompress, BMK_timedFnState_t* timeStateDecompress,
const void* srcBuffer, size_t srcSize,
const size_t* fileSizes, unsigned nbFiles,
const int cLevel, const ZSTD_compressionParameters* comprParams,
const void* dictBuffer, size_t dictBufferSize,
ZSTD_CCtx* ctx, ZSTD_DCtx* dctx,
int displayLevel, const char* displayName, const BMK_advancedParams_t* adv)
{
size_t const blockSize = ((adv->blockSize>=32 && (adv->mode != BMK_decodeOnly)) ? adv->blockSize : srcSize) + (!srcSize); /* avoid div by 0 */
BMK_return_t results;
size_t const loadedCompressedSize = srcSize;
size_t cSize = 0;
double ratio = 0.;
U32 nbBlocks;
/* checks */
if (!compressedBuffer || !resultBuffer || !blockTable)
EXM_THROW(31, "allocation error : not enough memory");
if(!ctx || !dctx)
EXM_THROW(31, "error: passed in null context");
EXM_THROW(31, BMK_return_t, "error: passed in null context");
/* init */
if (strlen(displayName)>17) displayName += strlen(displayName)-17; /* display last 17 characters */
if (g_nbWorkers==1) g_nbWorkers=0; /* prefer synchronous mode */
if (g_decodeOnly) { /* benchmark only decompression : source must be already compressed */
if (adv->mode == BMK_decodeOnly) { /* benchmark only decompression : source must be already compressed */
const char* srcPtr = (const char*)srcBuffer;
U64 totalDSize64 = 0;
U32 fileNb;
for (fileNb=0; fileNb<nbFiles; fileNb++) {
U64 const fSize64 = ZSTD_findDecompressedSize(srcPtr, fileSizes[fileNb]);
if (fSize64==0) EXM_THROW(32, "Impossible to determine original size ");
if (fSize64==0) EXM_THROW(32, BMK_return_t, "Impossible to determine original size ");
totalDSize64 += fSize64;
srcPtr += fileSizes[fileNb];
}
{ size_t const decodedSize = (size_t)totalDSize64;
if (totalDSize64 > decodedSize) EXM_THROW(32, "original size is too large"); /* size_t overflow */
free(resultBuffer);
resultBuffer = malloc(decodedSize);
if (!resultBuffer) EXM_THROW(33, "not enough memory");
if (!resultBuffer) {
EXM_THROW(33, BMK_return_t, "not enough memory");
}
if (totalDSize64 > decodedSize) {
free(resultBuffer);
EXM_THROW(32, BMK_return_t, "original size is too large"); /* size_t overflow */
}
cSize = srcSize;
srcSize = decodedSize;
ratio = (double)srcSize / (double)cSize;
} }
}
}
/* Init blockTable data */
/* Init data blocks */
{ const char* srcPtr = (const char*)srcBuffer;
char* cPtr = (char*)compressedBuffer;
char* resPtr = (char*)resultBuffer;
U32 fileNb;
for (nbBlocks=0, fileNb=0; fileNb<nbFiles; fileNb++) {
size_t remaining = fileSizes[fileNb];
U32 const nbBlocksforThisFile = g_decodeOnly ? 1 : (U32)((remaining + (blockSize-1)) / blockSize);
U32 const nbBlocksforThisFile = (adv->mode == BMK_decodeOnly) ? 1 : (U32)((remaining + (blockSize-1)) / blockSize);
U32 const blockEnd = nbBlocks + nbBlocksforThisFile;
for ( ; nbBlocks<blockEnd; nbBlocks++) {
size_t const thisBlockSize = MIN(remaining, blockSize);
blockTable[nbBlocks].srcPtr = (const void*)srcPtr;
blockTable[nbBlocks].srcSize = thisBlockSize;
blockTable[nbBlocks].cPtr = (void*)cPtr;
blockTable[nbBlocks].cRoom = g_decodeOnly ? thisBlockSize : ZSTD_compressBound(thisBlockSize);
blockTable[nbBlocks].cSize = blockTable[nbBlocks].cRoom;
blockTable[nbBlocks].resPtr = (void*)resPtr;
blockTable[nbBlocks].resSize = g_decodeOnly ? (size_t) ZSTD_findDecompressedSize(srcPtr, thisBlockSize) : thisBlockSize;
srcPtrs[nbBlocks] = (const void*)srcPtr;
srcSizes[nbBlocks] = thisBlockSize;
cPtrs[nbBlocks] = (void*)cPtr;
cSizes[nbBlocks] = (adv->mode == BMK_decodeOnly) ? thisBlockSize : ZSTD_compressBound(thisBlockSize);
resPtrs[nbBlocks] = (void*)resPtr;
resSizes[nbBlocks] = (adv->mode == BMK_decodeOnly) ? (size_t) ZSTD_findDecompressedSize(srcPtr, thisBlockSize) : thisBlockSize;
srcPtr += thisBlockSize;
cPtr += blockTable[nbBlocks].cRoom;
cPtr += cSizes[nbBlocks];
resPtr += thisBlockSize;
remaining -= thisBlockSize;
} } }
}
}
}
/* warmimg up memory */
if (g_decodeOnly) {
if (adv->mode == BMK_decodeOnly) {
memcpy(compressedBuffer, srcBuffer, loadedCompressedSize);
} else {
RDG_genBuffer(compressedBuffer, maxCompressedSize, 0.10, 0.50, 1);
}
/* Bench */
{ U64 fastestC = (U64)(-1LL), fastestD = (U64)(-1LL);
U64 const crcOrig = g_decodeOnly ? 0 : XXH64(srcBuffer, srcSize, 0);
UTIL_time_t coolTime;
U64 const maxTime = (g_nbSeconds * TIMELOOP_NANOSEC) + 1;
U32 nbDecodeLoops = (U32)((100 MB) / (srcSize+1)) + 1; /* initial conservative speed estimate */
U32 nbCompressionLoops = (U32)((2 MB) / (srcSize+1)) + 1; /* initial conservative speed estimate */
U64 totalCTime=0, totalDTime=0;
U32 cCompleted=g_decodeOnly, dCompleted=0;
{
U64 const crcOrig = (adv->mode == BMK_decodeOnly) ? 0 : XXH64(srcBuffer, srcSize, 0);
# define NB_MARKS 4
const char* const marks[NB_MARKS] = { " |", " /", " =", "\\" };
U32 markNb = 0;
coolTime = UTIL_getTime();
DISPLAYLEVEL(2, "\r%79s\r", "");
while (!cCompleted || !dCompleted) {
/* overheat protection */
if (UTIL_clockSpanMicro(coolTime) > ACTIVEPERIOD_MICROSEC) {
DISPLAYLEVEL(2, "\rcooling down ... \r");
UTIL_sleep(COOLPERIOD_SEC);
coolTime = UTIL_getTime();
}
if (!g_decodeOnly) {
/* Compression */
DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->\r", marks[markNb], displayName, (U32)srcSize);
if (!cCompleted) memset(compressedBuffer, 0xE5, maxCompressedSize); /* warm up and erase result buffer */
UTIL_sleepMilli(5); /* give processor time to other processes */
UTIL_waitForNextTick();
if (!cCompleted) { /* still some time to do compression tests */
U32 nbLoops = 0;
UTIL_time_t const clockStart = UTIL_getTime();
ZSTD_CCtx_setParameter(ctx, ZSTD_p_nbWorkers, g_nbWorkers);
ZSTD_CCtx_setParameter(ctx, ZSTD_p_compressionLevel, cLevel);
ZSTD_CCtx_setParameter(ctx, ZSTD_p_enableLongDistanceMatching, g_ldmFlag);
ZSTD_CCtx_setParameter(ctx, ZSTD_p_ldmMinMatch, g_ldmMinMatch);
ZSTD_CCtx_setParameter(ctx, ZSTD_p_ldmHashLog, g_ldmHashLog);
if (g_ldmBucketSizeLog != BMK_LDM_PARAM_NOTSET) {
ZSTD_CCtx_setParameter(ctx, ZSTD_p_ldmBucketSizeLog, g_ldmBucketSizeLog);
}
if (g_ldmHashEveryLog != BMK_LDM_PARAM_NOTSET) {
ZSTD_CCtx_setParameter(ctx, ZSTD_p_ldmHashEveryLog, g_ldmHashEveryLog);
}
ZSTD_CCtx_setParameter(ctx, ZSTD_p_windowLog, comprParams->windowLog);
ZSTD_CCtx_setParameter(ctx, ZSTD_p_hashLog, comprParams->hashLog);
ZSTD_CCtx_setParameter(ctx, ZSTD_p_chainLog, comprParams->chainLog);
ZSTD_CCtx_setParameter(ctx, ZSTD_p_searchLog, comprParams->searchLog);
ZSTD_CCtx_setParameter(ctx, ZSTD_p_minMatch, comprParams->searchLength);
ZSTD_CCtx_setParameter(ctx, ZSTD_p_targetLength, comprParams->targetLength);
ZSTD_CCtx_setParameter(ctx, ZSTD_p_compressionStrategy, comprParams->strategy);
ZSTD_CCtx_loadDictionary(ctx, dictBuffer, dictBufferSize);
if (!g_nbSeconds) nbCompressionLoops=1;
for (nbLoops=0; nbLoops<nbCompressionLoops; nbLoops++) {
U32 blockNb;
for (blockNb=0; blockNb<nbBlocks; blockNb++) {
#if 0 /* direct compression function, for occasional comparison */
ZSTD_parameters const params = ZSTD_getParams(cLevel, blockTable[blockNb].srcSize, dictBufferSize);
blockTable[blockNb].cSize = ZSTD_compress_advanced(ctx,
blockTable[blockNb].cPtr, blockTable[blockNb].cRoom,
blockTable[blockNb].srcPtr, blockTable[blockNb].srcSize,
dictBuffer, dictBufferSize,
params);
#else
size_t moreToFlush = 1;
ZSTD_outBuffer out;
ZSTD_inBuffer in;
in.src = blockTable[blockNb].srcPtr;
in.size = blockTable[blockNb].srcSize;
in.pos = 0;
out.dst = blockTable[blockNb].cPtr;
out.size = blockTable[blockNb].cRoom;
out.pos = 0;
while (moreToFlush) {
moreToFlush = ZSTD_compress_generic(ctx,
&out, &in, ZSTD_e_end);
if (ZSTD_isError(moreToFlush))
EXM_THROW(1, "ZSTD_compress_generic() error : %s",
ZSTD_getErrorName(moreToFlush));
}
blockTable[blockNb].cSize = out.pos;
#endif
} }
{ U64 const loopDuration = UTIL_clockSpanNano(clockStart);
if (loopDuration > 0) {
if (loopDuration < fastestC * nbCompressionLoops)
fastestC = loopDuration / nbCompressionLoops;
nbCompressionLoops = (U32)(TIMELOOP_NANOSEC / fastestC) + 1;
} else {
assert(nbCompressionLoops < 40000000); /* avoid overflow */
nbCompressionLoops *= 100;
}
totalCTime += loopDuration;
cCompleted = (totalCTime >= maxTime); /* end compression tests */
} }
cSize = 0;
{ U32 blockNb; for (blockNb=0; blockNb<nbBlocks; blockNb++) cSize += blockTable[blockNb].cSize; }
ratio = (double)srcSize / (double)cSize;
results.result.cSize = cSize;
markNb = (markNb+1) % NB_MARKS;
{ int const ratioAccuracy = (ratio < 10.) ? 3 : 2;
double const compressionSpeed = ((double)srcSize / fastestC) * 1000;
int const cSpeedAccuracy = (compressionSpeed < 10.) ? 2 : 1;
results.result.cSpeed = compressionSpeed * 1000000;
DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.*f),%6.*f MB/s\r",
marks[markNb], displayName, (U32)srcSize, (U32)cSize,
ratioAccuracy, ratio,
cSpeedAccuracy, compressionSpeed );
DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->\r", marks[markNb], displayName, (U32)srcSize);
{
BMK_initCCtxArgs cctxprep;
BMK_initDCtxArgs dctxprep;
cctxprep.ctx = ctx;
cctxprep.dictBuffer = dictBuffer;
cctxprep.dictBufferSize = dictBufferSize;
cctxprep.cLevel = cLevel;
cctxprep.comprParams = comprParams;
cctxprep.adv = adv;
dctxprep.dctx = dctx;
dctxprep.dictBuffer = dictBuffer;
dctxprep.dictBufferSize = dictBufferSize;
if(adv->loopMode == BMK_timeMode) {
BMK_customTimedReturn_t intermediateResultCompress;
BMK_customTimedReturn_t intermediateResultDecompress;
if(adv->mode == BMK_compressOnly) {
intermediateResultCompress.completed = 0;
intermediateResultDecompress.completed = 1;
} else if (adv->mode == BMK_decodeOnly) {
intermediateResultCompress.completed = 1;
intermediateResultDecompress.completed = 0;
} else { /* both */
intermediateResultCompress.completed = 0;
intermediateResultDecompress.completed = 0;
}
} /* if (!g_decodeOnly) */
#if 0 /* disable decompression test */
dCompleted=1;
(void)totalDTime; (void)fastestD; (void)crcOrig; /* unused when decompression disabled */
#else
/* Decompression */
if (!dCompleted) memset(resultBuffer, 0xD6, srcSize); /* warm result buffer */
UTIL_sleepMilli(5); /* give processor time to other processes */
UTIL_waitForNextTick();
if (!dCompleted) {
U32 nbLoops = 0;
ZSTD_DDict* const ddict = ZSTD_createDDict(dictBuffer, dictBufferSize);
UTIL_time_t const clockStart = UTIL_getTime();
if (!ddict) EXM_THROW(2, "ZSTD_createDDict() allocation failure");
if (!g_nbSeconds) nbDecodeLoops = 1;
for (nbLoops=0; nbLoops < nbDecodeLoops; nbLoops++) {
U32 blockNb;
for (blockNb=0; blockNb<nbBlocks; blockNb++) {
size_t const regenSize = ZSTD_decompress_usingDDict(dctx,
blockTable[blockNb].resPtr, blockTable[blockNb].resSize,
blockTable[blockNb].cPtr, blockTable[blockNb].cSize,
ddict);
if (ZSTD_isError(regenSize)) {
EXM_THROW(2, "ZSTD_decompress_usingDDict() failed on block %u of size %u : %s \n",
blockNb, (U32)blockTable[blockNb].cSize, ZSTD_getErrorName(regenSize));
while(!(intermediateResultCompress.completed && intermediateResultDecompress.completed)) {
if(!intermediateResultCompress.completed) {
intermediateResultCompress = BMK_benchFunctionTimed(timeStateCompress, &local_defaultCompress, (void*)ctx, &local_initCCtx, (void*)&cctxprep,
nbBlocks, srcPtrs, srcSizes, cPtrs, cSizes);
if(intermediateResultCompress.result.error) {
results.error = intermediateResultCompress.result.error;
return results;
}
ratio = (double)(srcSize / intermediateResultCompress.result.result.sumOfReturn);
{
int const ratioAccuracy = (ratio < 10.) ? 3 : 2;
double const compressionSpeed = ((double)srcSize / intermediateResultCompress.result.result.nanoSecPerRun) * 1000;
int const cSpeedAccuracy = (compressionSpeed < 10.) ? 2 : 1;
results.result.cSpeed = compressionSpeed * 1000000;
results.result.cSize = intermediateResultCompress.result.result.sumOfReturn;
ratio = (double)srcSize / results.result.cSize;
markNb = (markNb+1) % NB_MARKS;
DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.*f),%6.*f MB/s\r",
marks[markNb], displayName, (U32)srcSize, (U32)results.result.cSize,
ratioAccuracy, ratio,
cSpeedAccuracy, compressionSpeed);
}
blockTable[blockNb].resSize = regenSize;
} }
ZSTD_freeDDict(ddict);
{ U64 const loopDuration = UTIL_clockSpanNano(clockStart);
if (loopDuration > 0) {
if (loopDuration < fastestD * nbDecodeLoops)
fastestD = loopDuration / nbDecodeLoops;
nbDecodeLoops = (U32)(TIMELOOP_NANOSEC / fastestD) + 1;
} else {
assert(nbDecodeLoops < 40000000); /* avoid overflow */
nbDecodeLoops *= 100;
}
totalDTime += loopDuration;
dCompleted = (totalDTime >= maxTime);
} }
markNb = (markNb+1) % NB_MARKS;
{ int const ratioAccuracy = (ratio < 10.) ? 3 : 2;
double const compressionSpeed = ((double)srcSize / fastestC) * 1000;
int const cSpeedAccuracy = (compressionSpeed < 10.) ? 2 : 1;
double const decompressionSpeed = ((double)srcSize / fastestD) * 1000;
results.result.cSpeed = compressionSpeed * 1000000;
results.result.dSpeed = decompressionSpeed * 1000000;
DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.*f),%6.*f MB/s ,%6.1f MB/s \r",
marks[markNb], displayName, (U32)srcSize, (U32)cSize,
ratioAccuracy, ratio,
cSpeedAccuracy, compressionSpeed,
decompressionSpeed);
}
/* CRC Checking */
{ U64 const crcCheck = XXH64(resultBuffer, srcSize, 0);
if (!g_decodeOnly && (crcOrig!=crcCheck)) {
size_t u;
DISPLAY("!!! WARNING !!! %14s : Invalid Checksum : %x != %x \n", displayName, (unsigned)crcOrig, (unsigned)crcCheck);
for (u=0; u<srcSize; u++) {
if (((const BYTE*)srcBuffer)[u] != ((const BYTE*)resultBuffer)[u]) {
U32 segNb, bNb, pos;
size_t bacc = 0;
DISPLAY("Decoding error at pos %u ", (U32)u);
for (segNb = 0; segNb < nbBlocks; segNb++) {
if (bacc + blockTable[segNb].srcSize > u) break;
bacc += blockTable[segNb].srcSize;
}
pos = (U32)(u - bacc);
bNb = pos / (128 KB);
DISPLAY("(sample %u, block %u, pos %u) \n", segNb, bNb, pos);
if (u>5) {
int n;
DISPLAY("origin: ");
for (n=-5; n<0; n++) DISPLAY("%02X ", ((const BYTE*)srcBuffer)[u+n]);
DISPLAY(" :%02X: ", ((const BYTE*)srcBuffer)[u]);
for (n=1; n<3; n++) DISPLAY("%02X ", ((const BYTE*)srcBuffer)[u+n]);
DISPLAY(" \n");
DISPLAY("decode: ");
for (n=-5; n<0; n++) DISPLAY("%02X ", ((const BYTE*)resultBuffer)[u+n]);
DISPLAY(" :%02X: ", ((const BYTE*)resultBuffer)[u]);
for (n=1; n<3; n++) DISPLAY("%02X ", ((const BYTE*)resultBuffer)[u+n]);
DISPLAY(" \n");
}
break;
if(!intermediateResultDecompress.completed) {
intermediateResultDecompress = BMK_benchFunctionTimed(timeStateDecompress, &local_defaultDecompress, (void*)(dctx), &local_initDCtx, (void*)&dctxprep,
nbBlocks, (const void* const*)cPtrs, cSizes, resPtrs, resSizes);
if(intermediateResultDecompress.result.error) {
results.error = intermediateResultDecompress.result.error;
return results;
}
if (u==srcSize-1) { /* should never happen */
DISPLAY("no difference detected\n");
} }
break;
} } /* CRC Checking */
#endif
} /* for (testNb = 1; testNb <= (g_nbSeconds + !g_nbSeconds); testNb++) */
{
int const ratioAccuracy = (ratio < 10.) ? 3 : 2;
double const compressionSpeed = results.result.cSpeed / 1000000;
int const cSpeedAccuracy = (compressionSpeed < 10.) ? 2 : 1;
double const decompressionSpeed = ((double)srcSize / intermediateResultDecompress.result.result.nanoSecPerRun) * 1000;
results.result.dSpeed = decompressionSpeed * 1000000;
markNb = (markNb+1) % NB_MARKS;
DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.*f),%6.*f MB/s ,%6.1f MB/s \r",
marks[markNb], displayName, (U32)srcSize, (U32)results.result.cSize,
ratioAccuracy, ratio,
cSpeedAccuracy, compressionSpeed,
decompressionSpeed);
}
}
}
if (displayLevel == 1) { /* hidden display mode -q, used by python speed benchmark */
double const cSpeed = ((double)srcSize / fastestC) * 1000;
double const dSpeed = ((double)srcSize / fastestD) * 1000;
if (g_additionalParam)
DISPLAY("-%-3i%11i (%5.3f) %6.2f MB/s %6.1f MB/s %s (param=%d)\n", cLevel, (int)cSize, ratio, cSpeed, dSpeed, displayName, g_additionalParam);
else
DISPLAY("-%-3i%11i (%5.3f) %6.2f MB/s %6.1f MB/s %s\n", cLevel, (int)cSize, ratio, cSpeed, dSpeed, displayName);
} else {
if(adv->mode != BMK_decodeOnly) {
BMK_customReturn_t compressionResults = BMK_benchFunction(&local_defaultCompress, (void*)ctx, &local_initCCtx, (void*)&cctxprep,
nbBlocks, srcPtrs, srcSizes, cPtrs, cSizes, adv->nbSeconds);
if(compressionResults.error) {
results.error = compressionResults.error;
return results;
}
if(compressionResults.result.nanoSecPerRun == 0) {
results.result.cSpeed = 0;
} else {
results.result.cSpeed = (double)srcSize / compressionResults.result.nanoSecPerRun * TIMELOOP_NANOSEC;
}
results.result.cSize = compressionResults.result.sumOfReturn;
{
int const ratioAccuracy = (ratio < 10.) ? 3 : 2;
double const compressionSpeed = results.result.cSpeed / 1000000;
int const cSpeedAccuracy = (compressionSpeed < 10.) ? 2 : 1;
ratio = (double)srcSize / results.result.cSize;
markNb = (markNb+1) % NB_MARKS;
DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.*f),%6.*f MB/s\r",
marks[markNb], displayName, (U32)srcSize, (U32)results.result.cSize,
ratioAccuracy, ratio,
cSpeedAccuracy, compressionSpeed);
}
}
if(adv->mode != BMK_compressOnly) {
BMK_customReturn_t decompressionResults = BMK_benchFunction(
&local_defaultDecompress, (void*)(dctx),
&local_initDCtx, (void*)&dctxprep, nbBlocks,
(const void* const*)cPtrs, cSizes, resPtrs, resSizes,
adv->nbSeconds);
if(decompressionResults.error) {
results.error = decompressionResults.error;
return results;
}
if(decompressionResults.result.nanoSecPerRun == 0) {
results.result.dSpeed = 0;
} else {
results.result.dSpeed = (double)srcSize / decompressionResults.result.nanoSecPerRun * TIMELOOP_NANOSEC;
}
{ int const ratioAccuracy = (ratio < 10.) ? 3 : 2;
double const compressionSpeed = results.result.cSpeed / 1000000;
int const cSpeedAccuracy = (compressionSpeed < 10.) ? 2 : 1;
double const decompressionSpeed = ((double)srcSize / decompressionResults.result.nanoSecPerRun) * 1000;
results.result.dSpeed = decompressionSpeed * 1000000;
markNb = (markNb+1) % NB_MARKS;
DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.*f),%6.*f MB/s ,%6.1f MB/s \r",
marks[markNb], displayName, (U32)srcSize, (U32)results.result.cSize,
ratioAccuracy, ratio,
cSpeedAccuracy, compressionSpeed,
decompressionSpeed);
}
}
}
}
DISPLAYLEVEL(2, "%2i#\n", cLevel);
} /* Bench */
/* clean up */
free(blockTable);
free(compressedBuffer);
free(resultBuffer);
results.errorCode = 0;
/* CRC Checking */
{ U64 const crcCheck = XXH64(resultBuffer, srcSize, 0);
/* adv->mode == 0 -> compress + decompress */
if ((adv->mode == BMK_both) && (crcOrig!=crcCheck)) {
size_t u;
DISPLAY("!!! WARNING !!! %14s : Invalid Checksum : %x != %x \n", displayName, (unsigned)crcOrig, (unsigned)crcCheck);
for (u=0; u<srcSize; u++) {
if (((const BYTE*)srcBuffer)[u] != ((const BYTE*)resultBuffer)[u]) {
U32 segNb, bNb, pos;
size_t bacc = 0;
DISPLAY("Decoding error at pos %u ", (U32)u);
for (segNb = 0; segNb < nbBlocks; segNb++) {
if (bacc + srcSizes[segNb] > u) break;
bacc += srcSizes[segNb];
}
pos = (U32)(u - bacc);
bNb = pos / (128 KB);
DISPLAY("(sample %u, block %u, pos %u) \n", segNb, bNb, pos);
if (u>5) {
int n;
DISPLAY("origin: ");
for (n=-5; n<0; n++) DISPLAY("%02X ", ((const BYTE*)srcBuffer)[u+n]);
DISPLAY(" :%02X: ", ((const BYTE*)srcBuffer)[u]);
for (n=1; n<3; n++) DISPLAY("%02X ", ((const BYTE*)srcBuffer)[u+n]);
DISPLAY(" \n");
DISPLAY("decode: ");
for (n=-5; n<0; n++) DISPLAY("%02X ", ((const BYTE*)resultBuffer)[u+n]);
DISPLAY(" :%02X: ", ((const BYTE*)resultBuffer)[u]);
for (n=1; n<3; n++) DISPLAY("%02X ", ((const BYTE*)resultBuffer)[u+n]);
DISPLAY(" \n");
}
break;
}
if (u==srcSize-1) { /* should never happen */
DISPLAY("no difference detected\n");
}
}
}
} /* CRC Checking */
if (displayLevel == 1) { /* hidden display mode -q, used by python speed benchmark */
double const cSpeed = results.result.cSpeed / 1000000;
double const dSpeed = results.result.dSpeed / 1000000;
if (adv->additionalParam) {
DISPLAY("-%-3i%11i (%5.3f) %6.2f MB/s %6.1f MB/s %s (param=%d)\n", cLevel, (int)cSize, ratio, cSpeed, dSpeed, displayName, adv->additionalParam);
} else {
DISPLAY("-%-3i%11i (%5.3f) %6.2f MB/s %6.1f MB/s %s\n", cLevel, (int)cSize, ratio, cSpeed, dSpeed, displayName);
}
}
DISPLAYLEVEL(2, "%2i#\n", cLevel);
} /* Bench */
return results;
}
static void BMK_benchMemCtxless(const void* srcBuffer, size_t srcSize,
BMK_return_t BMK_benchMemAdvanced(const void* srcBuffer, size_t srcSize,
const size_t* fileSizes, unsigned nbFiles,
const int cLevel, const ZSTD_compressionParameters* comprParams,
const void* dictBuffer, size_t dictBufferSize,
ZSTD_CCtx* ctx, ZSTD_DCtx* dctx,
int displayLevel, const char* displayName, const BMK_advancedParams_t* adv)
{
size_t const blockSize = ((adv->blockSize>=32 && (adv->mode != BMK_decodeOnly)) ? adv->blockSize : srcSize) + (!srcSize) /* avoid div by 0 */ ;
U32 const maxNbBlocks = (U32) ((srcSize + (blockSize-1)) / blockSize) + nbFiles;
/* these are the blockTable parameters, just split up */
const void ** const srcPtrs = (const void** const)malloc(maxNbBlocks * sizeof(void*));
size_t* const srcSizes = (size_t* const)malloc(maxNbBlocks * sizeof(size_t));
void ** const cPtrs = (void** const)malloc(maxNbBlocks * sizeof(void*));
size_t* const cSizes = (size_t* const)malloc(maxNbBlocks * sizeof(size_t));
void ** const resPtrs = (void** const)malloc(maxNbBlocks * sizeof(void*));
size_t* const resSizes = (size_t* const)malloc(maxNbBlocks * sizeof(size_t));
const size_t maxCompressedSize = ZSTD_compressBound(srcSize) + (maxNbBlocks * 1024); /* add some room for safety */
void* compressedBuffer = malloc(maxCompressedSize);
void* resultBuffer = malloc(srcSize);
BMK_timedFnState_t* timeStateCompress = BMK_createTimeState(adv->nbSeconds);
BMK_timedFnState_t* timeStateDecompress = BMK_createTimeState(adv->nbSeconds);
BMK_return_t results;
int allocationincomplete = !compressedBuffer || !resultBuffer ||
!srcPtrs || !srcSizes || !cPtrs || !cSizes || !resPtrs || !resSizes;
if (!allocationincomplete) {
results = BMK_benchMemAdvancedNoAlloc(srcPtrs, srcSizes, cPtrs, cSizes,
resPtrs, resSizes, resultBuffer, compressedBuffer, maxCompressedSize, timeStateCompress, timeStateDecompress,
srcBuffer, srcSize, fileSizes, nbFiles, cLevel, comprParams,
dictBuffer, dictBufferSize, ctx, dctx, displayLevel, displayName, adv);
}
/* clean up */
BMK_freeTimeState(timeStateCompress);
BMK_freeTimeState(timeStateDecompress);
free(compressedBuffer);
free(resultBuffer);
free((void*)srcPtrs);
free(srcSizes);
free(cPtrs);
free(cSizes);
free(resPtrs);
free(resSizes);
if(allocationincomplete) {
EXM_THROW(31, BMK_return_t, "allocation error : not enough memory");
}
results.error = 0;
return results;
}
BMK_return_t BMK_benchMem(const void* srcBuffer, size_t srcSize,
const size_t* fileSizes, unsigned nbFiles,
const int cLevel, const ZSTD_compressionParameters* comprParams,
const void* dictBuffer, size_t dictBufferSize,
ZSTD_CCtx* ctx, ZSTD_DCtx* dctx,
int displayLevel, const char* displayName) {
const BMK_advancedParams_t adv = BMK_initAdvancedParams();
return BMK_benchMemAdvanced(srcBuffer, srcSize,
fileSizes, nbFiles,
cLevel, comprParams,
dictBuffer, dictBufferSize,
ctx, dctx,
displayLevel, displayName, &adv);
}
static BMK_return_t BMK_benchMemCtxless(const void* srcBuffer, size_t srcSize,
const size_t* fileSizes, unsigned nbFiles,
int cLevel, const ZSTD_compressionParameters* const comprParams,
const void* dictBuffer, size_t dictBufferSize,
int displayLevel, const char* displayName)
int displayLevel, const char* displayName,
const BMK_advancedParams_t* const adv)
{
BMK_return_t res;
ZSTD_CCtx* ctx = ZSTD_createCCtx();
ZSTD_DCtx* dctx = ZSTD_createDCtx();
if(ctx == NULL || dctx == NULL) {
EXM_THROW(12, "not enough memory for contexts");
EXM_THROW(12, BMK_return_t, "not enough memory for contexts");
}
BMK_benchMem(srcBuffer, srcSize,
res = BMK_benchMemAdvanced(srcBuffer, srcSize,
fileSizes, nbFiles,
cLevel, comprParams,
dictBuffer, dictBufferSize,
ctx, dctx,
displayLevel, displayName);
displayLevel, displayName, adv);
ZSTD_freeCCtx(ctx);
ZSTD_freeDCtx(dctx);
return res;
}
static size_t BMK_findMaxMem(U64 requiredMem)
@@ -544,44 +808,43 @@ static size_t BMK_findMaxMem(U64 requiredMem)
return (size_t)(requiredMem);
}
/* returns average stats over all range [cLevel, cLevelLast] */
static void BMK_benchCLevel(const void* srcBuffer, size_t benchedSize,
static BMK_return_t BMK_benchCLevel(const void* srcBuffer, size_t benchedSize,
const size_t* fileSizes, unsigned nbFiles,
const int cLevel, const int cLevelLast, const ZSTD_compressionParameters* comprParams,
const int cLevel, const ZSTD_compressionParameters* comprParams,
const void* dictBuffer, size_t dictBufferSize,
int displayLevel, const char* displayName)
int displayLevel, const char* displayName,
BMK_advancedParams_t const * const adv)
{
int l;
BMK_return_t res;
const char* pch = strrchr(displayName, '\\'); /* Windows */
if (!pch) pch = strrchr(displayName, '/'); /* Linux */
if (pch) displayName = pch+1;
if (g_realTime) {
if (adv->realTime) {
DISPLAYLEVEL(2, "Note : switching to real-time priority \n");
SET_REALTIME_PRIORITY;
}
if (displayLevel == 1 && !g_additionalParam)
DISPLAY("bench %s %s: input %u bytes, %u seconds, %u KB blocks\n", ZSTD_VERSION_STRING, ZSTD_GIT_COMMIT_STRING, (U32)benchedSize, g_nbSeconds, (U32)(g_blockSize>>10));
if (displayLevel == 1 && !adv->additionalParam)
DISPLAY("bench %s %s: input %u bytes, %u seconds, %u KB blocks\n", ZSTD_VERSION_STRING, ZSTD_GIT_COMMIT_STRING, (U32)benchedSize, adv->nbSeconds, (U32)(adv->blockSize>>10));
for (l=cLevel; l <= cLevelLast; l++) {
if (l==0) continue; /* skip level 0 */
BMK_benchMemCtxless(srcBuffer, benchedSize,
fileSizes, nbFiles,
l, comprParams,
dictBuffer, dictBufferSize,
displayLevel, displayName);
}
res = BMK_benchMemCtxless(srcBuffer, benchedSize,
fileSizes, nbFiles,
cLevel, comprParams,
dictBuffer, dictBufferSize,
displayLevel, displayName,
adv);
return;
return res;
}
/*! BMK_loadFiles() :
* Loads `buffer` with content of files listed within `fileNamesTable`.
* At most, fills `buffer` entirely. */
static void BMK_loadFiles(void* buffer, size_t bufferSize,
static int BMK_loadFiles(void* buffer, size_t bufferSize,
size_t* fileSizes, const char* const * const fileNamesTable,
unsigned nbFiles, int displayLevel)
{
@@ -601,44 +864,65 @@ static void BMK_loadFiles(void* buffer, size_t bufferSize,
continue;
}
f = fopen(fileNamesTable[n], "rb");
if (f==NULL) EXM_THROW(10, "impossible to open file %s", fileNamesTable[n]);
if (f==NULL) EXM_THROW_INT(10, "impossible to open file %s", fileNamesTable[n]);
DISPLAYUPDATE(2, "Loading %s... \r", fileNamesTable[n]);
if (fileSize > bufferSize-pos) fileSize = bufferSize-pos, nbFiles=n; /* buffer too small - stop after this file */
{ size_t const readSize = fread(((char*)buffer)+pos, 1, (size_t)fileSize, f);
if (readSize != (size_t)fileSize) EXM_THROW(11, "could not read %s", fileNamesTable[n]);
if (readSize != (size_t)fileSize) EXM_THROW_INT(11, "could not read %s", fileNamesTable[n]);
pos += readSize; }
fileSizes[n] = (size_t)fileSize;
totalSize += (size_t)fileSize;
fclose(f);
}
if (totalSize == 0) EXM_THROW(12, "no data to bench");
if (totalSize == 0) EXM_THROW_INT(12, "no data to bench");
return 0;
}
static void BMK_benchFileTable(const char* const * const fileNamesTable, unsigned const nbFiles,
const char* const dictFileName, int const cLevel, int const cLevelLast,
const ZSTD_compressionParameters* const compressionParams, int displayLevel)
BMK_return_t BMK_benchFilesAdvanced(const char* const * const fileNamesTable, unsigned const nbFiles,
const char* const dictFileName, int const cLevel,
const ZSTD_compressionParameters* const compressionParams,
int displayLevel, const BMK_advancedParams_t * const adv)
{
void* srcBuffer;
void* srcBuffer = NULL;
size_t benchedSize;
void* dictBuffer = NULL;
size_t dictBufferSize = 0;
size_t* const fileSizes = (size_t*)malloc(nbFiles * sizeof(size_t));
size_t* fileSizes = NULL;
BMK_return_t res;
U64 const totalSizeToLoad = UTIL_getTotalFileSize(fileNamesTable, nbFiles);
if (!fileSizes) EXM_THROW(12, "not enough memory for fileSizes");
if(!nbFiles) {
EXM_THROW(14, BMK_return_t, "No Files to Benchmark");
}
if (cLevel > ZSTD_maxCLevel()) {
EXM_THROW(15, BMK_return_t, "Invalid Compression Level");
}
fileSizes = (size_t*)calloc(nbFiles, sizeof(size_t));
if (!fileSizes) EXM_THROW(12, BMK_return_t, "not enough memory for fileSizes");
/* Load dictionary */
if (dictFileName != NULL) {
U64 const dictFileSize = UTIL_getFileSize(dictFileName);
if (dictFileSize > 64 MB)
EXM_THROW(10, "dictionary file %s too large", dictFileName);
if (dictFileSize > 64 MB) {
free(fileSizes);
EXM_THROW(10, BMK_return_t, "dictionary file %s too large", dictFileName);
}
dictBufferSize = (size_t)dictFileSize;
dictBuffer = malloc(dictBufferSize);
if (dictBuffer==NULL)
EXM_THROW(11, "not enough memory for dictionary (%u bytes)",
if (dictBuffer==NULL) {
free(fileSizes);
EXM_THROW(11, BMK_return_t, "not enough memory for dictionary (%u bytes)",
(U32)dictBufferSize);
BMK_loadFiles(dictBuffer, dictBufferSize, fileSizes, &dictFileName, 1, displayLevel);
}
{
int errorCode = BMK_loadFiles(dictBuffer, dictBufferSize, fileSizes, &dictFileName, 1, displayLevel);
if(errorCode) {
res.error = errorCode;
goto _cleanUp;
}
}
}
/* Memory allocation & restrictions */
@@ -647,96 +931,80 @@ static void BMK_benchFileTable(const char* const * const fileNamesTable, unsigne
if (benchedSize < totalSizeToLoad)
DISPLAY("Not enough memory; testing %u MB only...\n", (U32)(benchedSize >> 20));
srcBuffer = malloc(benchedSize);
if (!srcBuffer) EXM_THROW(12, "not enough memory");
if (!srcBuffer) {
free(dictBuffer);
free(fileSizes);
EXM_THROW(12, BMK_return_t, "not enough memory");
}
/* Load input buffer */
BMK_loadFiles(srcBuffer, benchedSize, fileSizes, fileNamesTable, nbFiles, displayLevel);
/* Bench */
if (g_separateFiles) {
const BYTE* srcPtr = (const BYTE*)srcBuffer;
U32 fileNb;
BMK_result_t* resultarray = (BMK_result_t*)malloc(sizeof(BMK_result_t) * nbFiles);
if(resultarray == NULL) EXM_THROW(12, "not enough memory");
for (fileNb=0; fileNb<nbFiles; fileNb++) {
size_t const fileSize = fileSizes[fileNb];
BMK_benchCLevel(srcPtr, fileSize,
fileSizes+fileNb, 1,
cLevel, cLevelLast, compressionParams,
dictBuffer, dictBufferSize,
displayLevel, fileNamesTable[fileNb]);
srcPtr += fileSize;
{
int errorCode = BMK_loadFiles(srcBuffer, benchedSize, fileSizes, fileNamesTable, nbFiles, displayLevel);
if(errorCode) {
res.error = errorCode;
goto _cleanUp;
}
} else {
}
/* Bench */
{
char mfName[20] = {0};
snprintf (mfName, sizeof(mfName), " %u files", nbFiles);
{ const char* const displayName = (nbFiles > 1) ? mfName : fileNamesTable[0];
BMK_benchCLevel(srcBuffer, benchedSize,
{
const char* const displayName = (nbFiles > 1) ? mfName : fileNamesTable[0];
res = BMK_benchCLevel(srcBuffer, benchedSize,
fileSizes, nbFiles,
cLevel, cLevelLast, compressionParams,
cLevel, compressionParams,
dictBuffer, dictBufferSize,
displayLevel, displayName);
displayLevel, displayName,
adv);
} }
/* clean up */
_cleanUp:
free(srcBuffer);
free(dictBuffer);
free(fileSizes);
return res;
}
static void BMK_syntheticTest(int cLevel, int cLevelLast, double compressibility,
BMK_return_t BMK_syntheticTest(int cLevel, double compressibility,
const ZSTD_compressionParameters* compressionParams,
int displayLevel)
int displayLevel, const BMK_advancedParams_t * const adv)
{
char name[20] = {0};
size_t benchedSize = 10000000;
void* const srcBuffer = malloc(benchedSize);
void* srcBuffer;
BMK_return_t res;
if (cLevel > ZSTD_maxCLevel()) {
EXM_THROW(15, BMK_return_t, "Invalid Compression Level");
}
/* Memory allocation */
if (!srcBuffer) EXM_THROW(21, "not enough memory");
srcBuffer = malloc(benchedSize);
if (!srcBuffer) EXM_THROW(21, BMK_return_t, "not enough memory");
/* Fill input buffer */
RDG_genBuffer(srcBuffer, benchedSize, compressibility, 0.0, 0);
/* Bench */
snprintf (name, sizeof(name), "Synthetic %2u%%", (unsigned)(compressibility*100));
BMK_benchCLevel(srcBuffer, benchedSize,
res = BMK_benchCLevel(srcBuffer, benchedSize,
&benchedSize, 1,
cLevel, cLevelLast, compressionParams,
cLevel, compressionParams,
NULL, 0,
displayLevel, name);
displayLevel, name, adv);
/* clean up */
free(srcBuffer);
return res;
}
static void BMK_benchFilesFull(const char** fileNamesTable, unsigned nbFiles,
const char* dictFileName,
int cLevel, int cLevelLast,
const ZSTD_compressionParameters* compressionParams, int displayLevel)
{
double const compressibility = (double)g_compressibilityDefault / 100;
if (cLevel > ZSTD_maxCLevel()) cLevel = ZSTD_maxCLevel();
if (cLevelLast > ZSTD_maxCLevel()) cLevelLast = ZSTD_maxCLevel();
if (cLevelLast < cLevel) cLevelLast = cLevel;
if (cLevelLast > cLevel)
DISPLAYLEVEL(2, "Benchmarking levels from %d to %d\n", cLevel, cLevelLast);
if (nbFiles == 0)
BMK_syntheticTest(cLevel, cLevelLast, compressibility, compressionParams, displayLevel);
else
BMK_benchFileTable(fileNamesTable, nbFiles, dictFileName, cLevel, cLevelLast, compressionParams, displayLevel);
}
int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles,
const char* dictFileName,
int cLevel, int cLevelLast,
const ZSTD_compressionParameters* compressionParams,
BMK_return_t BMK_benchFiles(const char* const * const fileNamesTable, unsigned const nbFiles,
const char* const dictFileName,
int const cLevel, const ZSTD_compressionParameters* const compressionParams,
int displayLevel) {
BMK_benchFilesFull(fileNamesTable, nbFiles, dictFileName, cLevel, cLevelLast, compressionParams, displayLevel);
return 0;
const BMK_advancedParams_t adv = BMK_initAdvancedParams();
return BMK_benchFilesAdvanced(fileNamesTable, nbFiles, dictFileName, cLevel, compressionParams, displayLevel, &adv);
}
programs/bench.h
+181 -23
View File
@@ -19,25 +19,121 @@ extern "C" {
#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_compressionParameters */
#include "zstd.h" /* ZSTD_compressionParameters */
/* Creates a struct of type typeName with an int type .error field
* and a .result field of some baseType. Functions with return
* typeName pass a successful result with .error = 0 and .result
* with the intended result, while returning an error will result
* in .error != 0.
*/
#define ERROR_STRUCT(baseType, typeName) typedef struct { \
baseType result; \
int error; \
} typeName
typedef struct {
size_t cSize;
double cSpeed; /* bytes / sec */
double dSpeed;
} BMK_result_t;
/* 0 = no Error */
typedef struct {
int errorCode;
BMK_result_t result;
} BMK_return_t;
ERROR_STRUCT(BMK_result_t, BMK_return_t);
/* called in cli */
int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles, const char* dictFileName,
int cLevel, int cLevelLast, const ZSTD_compressionParameters* compressionParams,
/* Loads files in fileNamesTable into memory, as well as a dictionary
* from dictFileName, and then uses benchMem */
/* fileNamesTable - name of files to benchmark
* nbFiles - number of files (size of fileNamesTable), must be > 0
* dictFileName - name of dictionary file to load
* cLevel - compression level to benchmark, errors if invalid
* compressionParams - basic compression Parameters
* displayLevel - what gets printed
* 0 : no display;
* 1 : errors;
* 2 : + result + interaction + warnings;
* 3 : + progression;
* 4 : + information
* return
* .error will give a nonzero error value if an error has occured
* .result - if .error = 0, .result will return the time taken to compression speed
* (.cSpeed), decompression speed (.dSpeed), and compressed size (.cSize) of the original
* file
*/
BMK_return_t BMK_benchFiles(const char* const * const fileNamesTable, unsigned const nbFiles,
const char* const dictFileName,
int const cLevel, const ZSTD_compressionParameters* const compressionParams,
int displayLevel);
/* basic benchmarking function, called in paramgrill
* ctx, dctx must be valid */
typedef enum {
BMK_timeMode = 0,
BMK_iterMode = 1
} BMK_loopMode_t;
typedef enum {
BMK_both = 0,
BMK_decodeOnly = 1,
BMK_compressOnly = 2
} BMK_mode_t;
typedef struct {
BMK_mode_t mode; /* 0: all, 1: compress only 2: decode only */
BMK_loopMode_t loopMode; /* if loopmode, then nbSeconds = nbLoops */
unsigned nbSeconds; /* default timing is in nbSeconds */
size_t blockSize; /* Maximum allowable size of a block*/
unsigned nbWorkers; /* multithreading */
unsigned realTime; /* real time priority */
int additionalParam; /* used by python speed benchmark */
unsigned ldmFlag; /* enables long distance matching */
unsigned ldmMinMatch; /* below: parameters for long distance matching, see zstd.1.md for meaning */
unsigned ldmHashLog;
unsigned ldmBucketSizeLog;
unsigned ldmHashEveryLog;
} BMK_advancedParams_t;
/* returns default parameters used by nonAdvanced functions */
BMK_advancedParams_t BMK_initAdvancedParams(void);
/* See benchFiles for normal parameter uses and return, see advancedParams_t for adv */
BMK_return_t BMK_benchFilesAdvanced(const char* const * const fileNamesTable, unsigned const nbFiles,
const char* const dictFileName,
int const cLevel, const ZSTD_compressionParameters* const compressionParams,
int displayLevel, const BMK_advancedParams_t* const adv);
/* called in cli */
/* Generates a sample with datagen with the compressibility argument*/
/* cLevel - compression level to benchmark, errors if invalid
* compressibility - determines compressibility of sample
* compressionParams - basic compression Parameters
* displayLevel - see benchFiles
* adv - see advanced_Params_t
* return
* .error will give a nonzero error value if an error has occured
* .result - if .error = 0, .result will return the time taken to compression speed
* (.cSpeed), decompression speed (.dSpeed), and compressed size (.cSize) of the original
* file
*/
BMK_return_t BMK_syntheticTest(int cLevel, double compressibility,
const ZSTD_compressionParameters* compressionParams,
int displayLevel, const BMK_advancedParams_t * const adv);
/* basic benchmarking function, called in paramgrill
* applies ZSTD_compress_generic() and ZSTD_decompress_generic() on data in srcBuffer
* with specific compression parameters specified by other arguments using benchFunction
* (cLevel, comprParams + adv in advanced Mode) */
/* srcBuffer - data source, expected to be valid compressed data if in Decode Only Mode
* srcSize - size of data in srcBuffer
* cLevel - compression level
* comprParams - basic compression parameters
* dictBuffer - a dictionary if used, null otherwise
* dictBufferSize - size of dictBuffer, 0 otherwise
* ctx - Compression Context (must be provided)
* dctx - Decompression Context (must be provided)
* diplayLevel - see BMK_benchFiles
* displayName - name used by display
* return
* .error will give a nonzero value if an error has occured
* .result - if .error = 0, will give the same results as benchFiles
* but for the data stored in srcBuffer
*/
BMK_return_t BMK_benchMem(const void* srcBuffer, size_t srcSize,
const size_t* fileSizes, unsigned nbFiles,
const int cLevel, const ZSTD_compressionParameters* comprParams,
@@ -45,20 +141,82 @@ BMK_return_t BMK_benchMem(const void* srcBuffer, size_t srcSize,
ZSTD_CCtx* ctx, ZSTD_DCtx* dctx,
int displayLevel, const char* displayName);
/* Set Parameters */
void BMK_setNbSeconds(unsigned nbLoops);
void BMK_setBlockSize(size_t blockSize);
void BMK_setNbWorkers(unsigned nbWorkers);
void BMK_setRealTime(unsigned priority);
void BMK_setNotificationLevel(unsigned level);
void BMK_setSeparateFiles(unsigned separate);
void BMK_setAdditionalParam(int additionalParam);
void BMK_setDecodeOnlyMode(unsigned decodeFlag);
void BMK_setLdmFlag(unsigned ldmFlag);
void BMK_setLdmMinMatch(unsigned ldmMinMatch);
void BMK_setLdmHashLog(unsigned ldmHashLog);
void BMK_setLdmBucketSizeLog(unsigned ldmBucketSizeLog);
void BMK_setLdmHashEveryLog(unsigned ldmHashEveryLog);
/* See benchMem for normal parameter uses and return, see advancedParams_t for adv */
BMK_return_t BMK_benchMemAdvanced(const void* srcBuffer, size_t srcSize,
const size_t* fileSizes, unsigned nbFiles,
const int cLevel, const ZSTD_compressionParameters* comprParams,
const void* dictBuffer, size_t dictBufferSize,
ZSTD_CCtx* ctx, ZSTD_DCtx* dctx,
int displayLevel, const char* displayName,
const BMK_advancedParams_t* adv);
typedef struct {
size_t sumOfReturn; /* sum of return values */
U64 nanoSecPerRun; /* time per iteration */
} BMK_customResult_t;
ERROR_STRUCT(BMK_customResult_t, BMK_customReturn_t);
typedef size_t (*BMK_benchFn_t)(const void*, size_t, void*, size_t, void*);
typedef size_t (*BMK_initFn_t)(void*);
/* This function times the execution of 2 argument functions, benchFn and initFn */
/* benchFn - (*benchFn)(srcBuffers[i], srcSizes[i], dstBuffers[i], dstCapacities[i], benchPayload)
* is run nbLoops times
* initFn - (*initFn)(initPayload) is run once per benchmark at the beginning. This argument can
* be NULL, in which case nothing is run.
* blockCount - number of blocks (size of srcBuffers, srcSizes, dstBuffers, dstCapacities)
* srcBuffers - an array of buffers to be operated on by benchFn
* srcSizes - an array of the sizes of above buffers
* dstBuffers - an array of buffers to be written into by benchFn
* dstCapacities - an array of the capacities of above buffers.
* nbLoops - defines number of times benchFn is run.
* return
* .error will give a nonzero value if ZSTD_isError() is nonzero for any of the return
* of the calls to initFn and benchFn, or if benchFunction errors internally
* .result - if .error = 0, then .result will contain the sum of all return values of
* benchFn on the first iteration through all of the blocks (.sumOfReturn) and also
* the time per run of benchFn (.nanoSecPerRun). For the former, this
* is generally intended to be used on functions which return the # of bytes written
* into dstBuffer, hence this value will be the total amount of bytes written to
* dstBuffer.
*/
BMK_customReturn_t BMK_benchFunction(
BMK_benchFn_t benchFn, void* benchPayload,
BMK_initFn_t initFn, void* initPayload,
size_t blockCount,
const void* const * const srcBuffers, const size_t* srcSizes,
void* const * const dstBuffers, const size_t* dstCapacities,
unsigned nbLoops);
/* state information needed to advance computation for benchFunctionTimed */
typedef struct BMK_timeState_t BMK_timedFnState_t;
/* initializes timeState object with desired number of seconds */
BMK_timedFnState_t* BMK_createTimeState(unsigned nbSeconds);
/* resets existing timeState object */
void BMK_resetTimeState(BMK_timedFnState_t*, unsigned nbSeconds);
/* deletes timeState object */
void BMK_freeTimeState(BMK_timedFnState_t* state);
typedef struct {
BMK_customReturn_t result;
int completed;
} BMK_customTimedReturn_t;
/*
* Benchmarks custom functions like BMK_benchFunction(), but runs for nbSeconds seconds rather than a fixed number of loops
* arguments mostly the same other than BMK_benchFunction()
* Usage - benchFunctionTimed will return in approximately one second. Keep calling BMK_benchFunctionTimed() until the return's completed field = 1.
* to continue updating intermediate result. Intermediate return values are returned by the function.
*/
BMK_customTimedReturn_t BMK_benchFunctionTimed(BMK_timedFnState_t* cont,
BMK_benchFn_t benchFn, void* benchPayload,
BMK_initFn_t initFn, void* initPayload,
size_t blockCount,
const void* const * const srcBlockBuffers, const size_t* srcBlockSizes,
void* const * const dstBlockBuffers, const size_t* dstBlockCapacities);
#endif /* BENCH_H_121279284357 */
programs/dibio.c
+2 -1
View File
@@ -323,7 +323,8 @@ int DiB_trainFromFiles(const char* dictFileName, unsigned maxDictSize,
srcBuffer, sampleSizes, fs.nbSamples,
coverParams);
if (!ZDICT_isError(dictSize)) {
DISPLAYLEVEL(2, "k=%u\nd=%u\nsteps=%u\n", coverParams->k, coverParams->d, coverParams->steps);
unsigned splitPercentage = (unsigned)(coverParams->splitPoint * 100);
DISPLAYLEVEL(2, "k=%u\nd=%u\nsteps=%u\nsplit=%u\n", coverParams->k, coverParams->d, coverParams->steps, splitPercentage);
}
} else {
dictSize = ZDICT_trainFromBuffer_cover(dictBuffer, maxDictSize, srcBuffer,
programs/fileio.c
+10 -6
View File
@@ -2017,21 +2017,25 @@ static int FIO_listFile(fileInfo_t* total, const char* inFileName, int displayLe
}
int FIO_listMultipleFiles(unsigned numFiles, const char** filenameTable, int displayLevel){
if (!IS_CONSOLE(stdin)) {
DISPLAYOUT("zstd: --list does not support reading from standard input\n");
return 1;
unsigned u;
for (u=0; u<numFiles;u++) {
if (!strcmp (filenameTable[u], stdinmark)) {
DISPLAYOUT("zstd: --list does not support reading from standard input\n");
return 1;
}
}
if (numFiles == 0) {
if (!IS_CONSOLE(stdin)) {
DISPLAYOUT("zstd: --list does not support reading from standard input\n");
}
DISPLAYOUT("No files given\n");
return 0;
return 1;
}
if (displayLevel <= 2) {
DISPLAYOUT("Frames Skips Compressed Uncompressed Ratio Check Filename\n");
}
{ int error = 0;
unsigned u;
fileInfo_t total;
memset(&total, 0, sizeof(total));
total.usesCheck = 1;
programs/zstd.1
+3 -2
View File
@@ -217,8 +217,9 @@ Split input files in blocks of size # (default: no split)
A dictionary ID is a locally unique ID that a decoder can use to verify it is using the right dictionary\. By default, zstd will create a 4\-bytes random number ID\. It\'s possible to give a precise number instead\. Short numbers have an advantage : an ID < 256 will only need 1 byte in the compressed frame header, and an ID < 65536 will only need 2 bytes\. This compares favorably to 4 bytes default\. However, it\'s up to the dictionary manager to not assign twice the same ID to 2 different dictionaries\.
.
.TP
\fB\-\-train\-cover[=k#,d=#,steps=#]\fR
Select parameters for the default dictionary builder algorithm named cover\. If \fId\fR is not specified, then it tries \fId\fR = 6 and \fId\fR = 8\. If \fIk\fR is not specified, then it tries \fIsteps\fR values in the range [50, 2000]\. If \fIsteps\fR is not specified, then the default value of 40 is used\. Requires that \fId\fR <= \fIk\fR\.
\fB\-\-train\-cover[=k#,d=#,steps=#,split=#]\fR
Select parameters for the default dictionary builder algorithm named cover\. If \fId\fR is not specified, then it tries \fId\fR = 6 and \fId\fR = 8\. If \fIk\fR is not specified, then it tries \fIsteps\fR values in the range [50, 2000]\. If \fIsteps\fR is not specified, then the default value of 40 is used\. If \fIsplit\fR is not specified or \fIsplit\fR <= 0, then the default value of 100 is used\. If \fIsplit\fR is 100, all input samples are used for both training and testing
to find optimal _d_ and _k_ to build dictionary.Requires that \fId\fR <= \fIk\fR\.
.
.IP
Selects segments of size \fIk\fR with highest score to put in the dictionary\. The score of a segment is computed by the sum of the frequencies of all the subsegments of size \fId\fR\. Generally \fId\fR should be in the range [6, 8], occasionally up to 16, but the algorithm will run faster with d <= \fI8\fR\. Good values for \fIk\fR vary widely based on the input data, but a safe range is [2 * \fId\fR, 2000]\. Supports multithreading if \fBzstd\fR is compiled with threading support\.
programs/zstd.1.md
+6 -1
View File
@@ -223,11 +223,12 @@ Compression of small files similar to the sample set will be greatly improved.
This compares favorably to 4 bytes default.
However, it's up to the dictionary manager to not assign twice the same ID to
2 different dictionaries.
* `--train-cover[=k#,d=#,steps=#]`:
* `--train-cover[=k#,d=#,steps=#,split=#]`:
Select parameters for the default dictionary builder algorithm named cover.
If _d_ is not specified, then it tries _d_ = 6 and _d_ = 8.
If _k_ is not specified, then it tries _steps_ values in the range [50, 2000].
If _steps_ is not specified, then the default value of 40 is used.
If _split_ is not specified or split <= 0, then the default value of 100 is used.
Requires that _d_ <= _k_.
Selects segments of size _k_ with highest score to put in the dictionary.
@@ -237,6 +238,8 @@ Compression of small files similar to the sample set will be greatly improved.
algorithm will run faster with d <= _8_.
Good values for _k_ vary widely based on the input data, but a safe range is
[2 * _d_, 2000].
If _split_ is 100, all input samples are used for both training and testing
to find optimal _d_ and _k_ to build dictionary.
Supports multithreading if `zstd` is compiled with threading support.
Examples:
@@ -249,6 +252,8 @@ Compression of small files similar to the sample set will be greatly improved.
`zstd --train-cover=k=50 FILEs`
`zstd --train-cover=k=50,split=60 FILEs`
* `--train-legacy[=selectivity=#]`:
Use legacy dictionary builder algorithm with the given dictionary
_selectivity_ (default: 9).
programs/zstdcli.c
+60 -18
View File
@@ -32,7 +32,7 @@
#include <errno.h> /* errno */
#include "fileio.h" /* stdinmark, stdoutmark, ZSTD_EXTENSION */
#ifndef ZSTD_NOBENCH
# include "bench.h" /* BMK_benchFiles, BMK_SetNbSeconds */
# include "bench.h" /* BMK_benchFiles */
#endif
#ifndef ZSTD_NODICT
# include "dibio.h" /* ZDICT_cover_params_t, DiB_trainFromFiles() */
@@ -84,6 +84,7 @@ static U32 g_ldmMinMatch = 0;
static U32 g_ldmHashEveryLog = LDM_PARAM_DEFAULT;
static U32 g_ldmBucketSizeLog = LDM_PARAM_DEFAULT;
#define DEFAULT_SPLITPOINT 1.0
/*-************************************
* Display Macros
@@ -170,7 +171,7 @@ static int usage_advanced(const char* programName)
DISPLAY( "\n");
DISPLAY( "Dictionary builder : \n");
DISPLAY( "--train ## : create a dictionary from a training set of files \n");
DISPLAY( "--train-cover[=k=#,d=#,steps=#] : use the cover algorithm with optional args\n");
DISPLAY( "--train-cover[=k=#,d=#,steps=#,split=#] : use the cover algorithm with optional args\n");
DISPLAY( "--train-legacy[=s=#] : use the legacy algorithm with selectivity (default: %u)\n", g_defaultSelectivityLevel);
DISPLAY( " -o file : `file` is dictionary name (default: %s) \n", g_defaultDictName);
DISPLAY( "--maxdict=# : limit dictionary to specified size (default: %u) \n", g_defaultMaxDictSize);
@@ -282,10 +283,15 @@ static unsigned parseCoverParameters(const char* stringPtr, ZDICT_cover_params_t
if (longCommandWArg(&stringPtr, "k=")) { params->k = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
if (longCommandWArg(&stringPtr, "d=")) { params->d = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
if (longCommandWArg(&stringPtr, "steps=")) { params->steps = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
if (longCommandWArg(&stringPtr, "split=")) {
unsigned splitPercentage = readU32FromChar(&stringPtr);
params->splitPoint = (double)splitPercentage / 100.0;
if (stringPtr[0]==',') { stringPtr++; continue; } else break;
}
return 0;
}
if (stringPtr[0] != 0) return 0;
DISPLAYLEVEL(4, "cover: k=%u\nd=%u\nsteps=%u\n", params->k, params->d, params->steps);
DISPLAYLEVEL(4, "cover: k=%u\nd=%u\nsteps=%u\nsplit=%u\n", params->k, params->d, params->steps, (unsigned)(params->splitPoint * 100));
return 1;
}
@@ -310,6 +316,7 @@ static ZDICT_cover_params_t defaultCoverParams(void)
memset(&params, 0, sizeof(params));
params.d = 8;
params.steps = 4;
params.splitPoint = DEFAULT_SPLITPOINT;
return params;
}
#endif
@@ -398,6 +405,8 @@ int main(int argCount, const char* argv[])
setRealTimePrio = 0,
singleThread = 0,
ultra=0;
double compressibility = 0.5;
BMK_advancedParams_t adv = BMK_initAdvancedParams();
unsigned bench_nbSeconds = 3; /* would be better if this value was synchronized from bench */
size_t blockSize = 0;
zstd_operation_mode operation = zom_compress;
@@ -441,7 +450,7 @@ int main(int argCount, const char* argv[])
#endif
/* preset behaviors */
if (exeNameMatch(programName, ZSTD_ZSTDMT)) nbWorkers=0;
if (exeNameMatch(programName, ZSTD_ZSTDMT)) nbWorkers=0, singleThread=0;
if (exeNameMatch(programName, ZSTD_UNZSTD)) operation=zom_decompress;
if (exeNameMatch(programName, ZSTD_CAT)) { operation=zom_decompress; forceStdout=1; FIO_overwriteMode(); outFileName=stdoutmark; g_displayLevel=1; } /* supports multiple formats */
if (exeNameMatch(programName, ZSTD_ZCAT)) { operation=zom_decompress; forceStdout=1; FIO_overwriteMode(); outFileName=stdoutmark; g_displayLevel=1; } /* behave like zcat, also supports multiple formats */
@@ -609,7 +618,7 @@ int main(int argCount, const char* argv[])
/* Decoding */
case 'd':
#ifndef ZSTD_NOBENCH
BMK_setDecodeOnlyMode(1);
adv.mode = BMK_decodeOnly;
if (operation==zom_bench) { argument++; break; } /* benchmark decode (hidden option) */
#endif
operation=zom_decompress; argument++; break;
@@ -702,11 +711,19 @@ int main(int argCount, const char* argv[])
case 'p': argument++;
#ifndef ZSTD_NOBENCH
if ((*argument>='0') && (*argument<='9')) {
BMK_setAdditionalParam(readU32FromChar(&argument));
adv.additionalParam = (int)readU32FromChar(&argument);
} else
#endif
main_pause=1;
break;
/* Select compressibility of synthetic sample */
case 'P':
{ argument++;
compressibility = (double)readU32FromChar(&argument) / 100;
}
break;
/* unknown command */
default : CLEAN_RETURN(badusage(programName));
}
@@ -764,7 +781,7 @@ int main(int argCount, const char* argv[])
DISPLAYLEVEL(3, "Note: %d physical core(s) detected \n", nbWorkers);
}
#else
(void)singleThread;
(void)singleThread; (void)nbWorkers;
#endif
#ifdef UTIL_HAS_CREATEFILELIST
@@ -807,21 +824,46 @@ int main(int argCount, const char* argv[])
/* Check if benchmark is selected */
if (operation==zom_bench) {
#ifndef ZSTD_NOBENCH
BMK_setSeparateFiles(separateFiles);
BMK_setBlockSize(blockSize);
BMK_setNbWorkers(nbWorkers);
BMK_setRealTime(setRealTimePrio);
BMK_setNbSeconds(bench_nbSeconds);
BMK_setLdmFlag(ldmFlag);
BMK_setLdmMinMatch(g_ldmMinMatch);
BMK_setLdmHashLog(g_ldmHashLog);
adv.blockSize = blockSize;
adv.nbWorkers = nbWorkers;
adv.realTime = setRealTimePrio;
adv.nbSeconds = bench_nbSeconds;
adv.ldmFlag = ldmFlag;
adv.ldmMinMatch = g_ldmMinMatch;
adv.ldmHashLog = g_ldmHashLog;
if (g_ldmBucketSizeLog != LDM_PARAM_DEFAULT) {
BMK_setLdmBucketSizeLog(g_ldmBucketSizeLog);
adv.ldmBucketSizeLog = g_ldmBucketSizeLog;
}
if (g_ldmHashEveryLog != LDM_PARAM_DEFAULT) {
BMK_setLdmHashEveryLog(g_ldmHashEveryLog);
adv.ldmHashEveryLog = g_ldmHashEveryLog;
}
BMK_benchFiles(filenameTable, filenameIdx, dictFileName, cLevel, cLevelLast, &compressionParams, g_displayLevel);
if (cLevel > ZSTD_maxCLevel()) cLevel = ZSTD_maxCLevel();
if (cLevelLast > ZSTD_maxCLevel()) cLevelLast = ZSTD_maxCLevel();
if (cLevelLast < cLevel) cLevelLast = cLevel;
if (cLevelLast > cLevel)
DISPLAYLEVEL(2, "Benchmarking levels from %d to %d\n", cLevel, cLevelLast);
if(filenameIdx) {
if(separateFiles) {
unsigned i;
for(i = 0; i < filenameIdx; i++) {
int c;
DISPLAYLEVEL(2, "Benchmarking %s \n", filenameTable[i]);
for(c = cLevel; c <= cLevelLast; c++) {
BMK_benchFilesAdvanced(&filenameTable[i], 1, dictFileName, c, &compressionParams, g_displayLevel, &adv);
}
}
} else {
for(; cLevel <= cLevelLast; cLevel++) {
BMK_benchFilesAdvanced(filenameTable, filenameIdx, dictFileName, cLevel, &compressionParams, g_displayLevel, &adv);
}
}
} else {
for(; cLevel <= cLevelLast; cLevel++) {
BMK_syntheticTest(cLevel, compressibility, &compressionParams, g_displayLevel, &adv);
}
}
#else
(void)bench_nbSeconds; (void)blockSize; (void)setRealTimePrio; (void)separateFiles;
#endif
tests/Makefile
+15 -17
View File
@@ -88,13 +88,8 @@ allnothread: fullbench fuzzer paramgrill datagen decodecorpus
dll: fuzzer-dll zstreamtest-dll
zstd:
$(MAKE) -C $(PRGDIR) $@ MOREFLAGS+="$(DEBUGFLAGS)"
zstd32:
$(MAKE) -C $(PRGDIR) $@ MOREFLAGS+="$(DEBUGFLAGS)"
zstd-nolegacy:
PHONY: zstd zstd32 zstd-nolegacy # must be phony, only external makefile knows how to build them, or if they need an update
zstd zstd32 zstd-nolegacy:
$(MAKE) -C $(PRGDIR) $@ MOREFLAGS+="$(DEBUGFLAGS)"
gzstd:
@@ -133,7 +128,7 @@ fullbench fullbench32 : CPPFLAGS += $(MULTITHREAD_CPP)
fullbench fullbench32 : LDFLAGS += $(MULTITHREAD_LD)
fullbench fullbench32 : DEBUGFLAGS = # turn off assert() for speed measurements
fullbench fullbench32 : $(ZSTD_FILES)
fullbench fullbench32 : $(PRGDIR)/datagen.c fullbench.c
fullbench fullbench32 : $(PRGDIR)/datagen.c $(PRGDIR)/bench.c fullbench.c
$(CC) $(FLAGS) $^ -o $@$(EXT)
fullbench-lib : zstd-staticLib
@@ -245,13 +240,14 @@ checkTag: checkTag.c $(ZSTDDIR)/zstd.h
clean:
$(MAKE) -C $(ZSTDDIR) clean
$(MAKE) -C $(PRGDIR) clean
@$(RM) -fR $(TESTARTEFACT)
@$(RM) -f core *.o tmp* result* *.gcda dictionary *.zst \
$(PRGDIR)/zstd$(EXT) $(PRGDIR)/zstd32$(EXT) \
fullbench$(EXT) fullbench32$(EXT) \
fullbench-lib$(EXT) fullbench-dll$(EXT) \
fuzzer$(EXT) fuzzer32$(EXT) zbufftest$(EXT) zbufftest32$(EXT) \
fuzzer-dll$(EXT) zstreamtest-dll$(EXT) zbufftest-dll$(EXT)\
fuzzer-dll$(EXT) zstreamtest-dll$(EXT) zbufftest-dll$(EXT) \
zstreamtest$(EXT) zstreamtest32$(EXT) \
datagen$(EXT) paramgrill$(EXT) roundTripCrash$(EXT) longmatch$(EXT) \
symbols$(EXT) invalidDictionaries$(EXT) legacy$(EXT) poolTests$(EXT) \
@@ -301,11 +297,6 @@ endif
list:
@$(MAKE) -pRrq -f $(lastword $(MAKEFILE_LIST)) : 2>/dev/null | awk -v RS= -F: '/^# File/,/^# Finished Make data base/ {if ($$1 !~ "^[#.]") {print $$1}}' | sort | egrep -v -e '^[^[:alnum:]]' -e '^$@$$' | xargs
.PHONY: zstd-playTests
zstd-playTests: datagen
file $(ZSTD)
ZSTD="$(QEMU_SYS) $(ZSTD)" ./playTests.sh $(ZSTDRTTEST)
.PHONY: shortest
shortest: ZSTDRTTEST=
shortest: test-zstd
@@ -323,14 +314,21 @@ test32: test-zstd32 test-fullbench32 test-fuzzer32 test-zstream32
test-all: test test32 valgrindTest test-decodecorpus-cli
.PHONY: test-zstd test-zstd32 test-zstd-nolegacy
test-zstd: ZSTD = $(PRGDIR)/zstd
test-zstd: zstd zstd-playTests
test-zstd: zstd
test-zstd32: ZSTD = $(PRGDIR)/zstd32
test-zstd32: zstd32 zstd-playTests
test-zstd32: zstd32
test-zstd-nolegacy: ZSTD = $(PRGDIR)/zstd-nolegacy
test-zstd-nolegacy: zstd-nolegacy zstd-playTests
test-zstd-nolegacy: zstd-nolegacy
test-zstd test-zstd32 test-zstd-nolegacy: datagen
file $(ZSTD)
ZSTD="$(QEMU_SYS) $(ZSTD)" ./playTests.sh $(ZSTDRTTEST)
test-gzstd: gzstd
$(PRGDIR)/zstd -f README.md test-zstd-speed.py
tests/fullbench.c
+196 -105
View File
@@ -30,6 +30,7 @@
#include "zstd.h" /* ZSTD_versionString */
#include "util.h" /* time functions */
#include "datagen.h"
#include "bench.h" /* CustomBench*/
/*_************************************
@@ -45,6 +46,8 @@
#define KNUTH 2654435761U
#define MAX_MEM (1984 MB)
#define DEFAULT_CLEVEL 1
#define COMPRESSIBILITY_DEFAULT 0.50
static const size_t g_sampleSize = 10000000;
@@ -89,18 +92,63 @@ static size_t BMK_findMaxMem(U64 requiredMem)
return (size_t) requiredMem;
}
/*_*******************************************************
* Argument Parsing
*********************************************************/
#define ERROR_OUT(msg) { DISPLAY("%s \n", msg); exit(1); }
static unsigned readU32FromChar(const char** stringPtr)
{
const char errorMsg[] = "error: numeric value too large";
unsigned result = 0;
while ((**stringPtr >='0') && (**stringPtr <='9')) {
unsigned const max = (((unsigned)(-1)) / 10) - 1;
if (result > max) ERROR_OUT(errorMsg);
result *= 10, result += **stringPtr - '0', (*stringPtr)++ ;
}
if ((**stringPtr=='K') || (**stringPtr=='M')) {
unsigned const maxK = ((unsigned)(-1)) >> 10;
if (result > maxK) ERROR_OUT(errorMsg);
result <<= 10;
if (**stringPtr=='M') {
if (result > maxK) ERROR_OUT(errorMsg);
result <<= 10;
}
(*stringPtr)++; /* skip `K` or `M` */
if (**stringPtr=='i') (*stringPtr)++;
if (**stringPtr=='B') (*stringPtr)++;
}
return result;
}
static unsigned longCommandWArg(const char** stringPtr, const char* longCommand)
{
size_t const comSize = strlen(longCommand);
int const result = !strncmp(*stringPtr, longCommand, comSize);
if (result) *stringPtr += comSize;
return result;
}
/*_*******************************************************
* Benchmark wrappers
*********************************************************/
size_t local_ZSTD_compress(void* dst, size_t dstSize, void* buff2, const void* src, size_t srcSize)
static ZSTD_CCtx* g_zcc = NULL;
size_t local_ZSTD_compress(const void* src, size_t srcSize, void* dst, size_t dstSize, void* buff2)
{
(void)buff2;
return ZSTD_compress(dst, dstSize, src, srcSize, 1);
ZSTD_parameters p;
ZSTD_frameParameters f = {1 /* contentSizeHeader*/, 0, 0};
p.fParams = f;
p.cParams = *(ZSTD_compressionParameters*)buff2;
return ZSTD_compress_advanced (g_zcc,dst, dstSize, src, srcSize, NULL ,0, p);
//return ZSTD_compress(dst, dstSize, src, srcSize, cLevel);
}
static size_t g_cSize = 0;
size_t local_ZSTD_decompress(void* dst, size_t dstSize, void* buff2, const void* src, size_t srcSize)
size_t local_ZSTD_decompress(const void* src, size_t srcSize, void* dst, size_t dstSize, void* buff2)
{
(void)src; (void)srcSize;
return ZSTD_decompress(dst, dstSize, buff2, g_cSize);
@@ -110,14 +158,14 @@ static ZSTD_DCtx* g_zdc = NULL;
#ifndef ZSTD_DLL_IMPORT
extern size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* ctx, const void* src, size_t srcSize);
size_t local_ZSTD_decodeLiteralsBlock(void* dst, size_t dstSize, void* buff2, const void* src, size_t srcSize)
size_t local_ZSTD_decodeLiteralsBlock(const void* src, size_t srcSize, void* dst, size_t dstSize, void* buff2)
{
(void)src; (void)srcSize; (void)dst; (void)dstSize;
return ZSTD_decodeLiteralsBlock((ZSTD_DCtx*)g_zdc, buff2, g_cSize);
}
extern size_t ZSTD_decodeSeqHeaders(ZSTD_DCtx* dctx, int* nbSeq, const void* src, size_t srcSize);
size_t local_ZSTD_decodeSeqHeaders(void* dst, size_t dstSize, void* buff2, const void* src, size_t srcSize)
size_t local_ZSTD_decodeSeqHeaders(const void* src, size_t srcSize, void* dst, size_t dstSize, void* buff2)
{
int nbSeq;
(void)src; (void)srcSize; (void)dst; (void)dstSize;
@@ -126,12 +174,15 @@ size_t local_ZSTD_decodeSeqHeaders(void* dst, size_t dstSize, void* buff2, const
#endif
static ZSTD_CStream* g_cstream= NULL;
size_t local_ZSTD_compressStream(void* dst, size_t dstCapacity, void* buff2, const void* src, size_t srcSize)
size_t local_ZSTD_compressStream(const void* src, size_t srcSize, void* dst, size_t dstCapacity, void* buff2)
{
ZSTD_outBuffer buffOut;
ZSTD_inBuffer buffIn;
(void)buff2;
ZSTD_initCStream(g_cstream, 1);
ZSTD_parameters p;
ZSTD_frameParameters f = {1 /* contentSizeHeader*/, 0, 0};
p.fParams = f;
p.cParams = *(ZSTD_compressionParameters*)buff2;
ZSTD_initCStream_advanced(g_cstream, NULL, 0, p, ZSTD_CONTENTSIZE_UNKNOWN);
buffOut.dst = dst;
buffOut.size = dstCapacity;
buffOut.pos = 0;
@@ -143,12 +194,11 @@ size_t local_ZSTD_compressStream(void* dst, size_t dstCapacity, void* buff2, con
return buffOut.pos;
}
static size_t local_ZSTD_compress_generic_end(void* dst, size_t dstCapacity, void* buff2, const void* src, size_t srcSize)
static size_t local_ZSTD_compress_generic_end(const void* src, size_t srcSize, void* dst, size_t dstCapacity, void* buff2)
{
ZSTD_outBuffer buffOut;
ZSTD_inBuffer buffIn;
(void)buff2;
ZSTD_CCtx_setParameter(g_cstream, ZSTD_p_compressionLevel, 1);
buffOut.dst = dst;
buffOut.size = dstCapacity;
buffOut.pos = 0;
@@ -159,12 +209,11 @@ static size_t local_ZSTD_compress_generic_end(void* dst, size_t dstCapacity, voi
return buffOut.pos;
}
static size_t local_ZSTD_compress_generic_continue(void* dst, size_t dstCapacity, void* buff2, const void* src, size_t srcSize)
static size_t local_ZSTD_compress_generic_continue(const void* src, size_t srcSize, void* dst, size_t dstCapacity, void* buff2)
{
ZSTD_outBuffer buffOut;
ZSTD_inBuffer buffIn;
(void)buff2;
ZSTD_CCtx_setParameter(g_cstream, ZSTD_p_compressionLevel, 1);
buffOut.dst = dst;
buffOut.size = dstCapacity;
buffOut.pos = 0;
@@ -176,12 +225,11 @@ static size_t local_ZSTD_compress_generic_continue(void* dst, size_t dstCapacity
return buffOut.pos;
}
static size_t local_ZSTD_compress_generic_T2_end(void* dst, size_t dstCapacity, void* buff2, const void* src, size_t srcSize)
static size_t local_ZSTD_compress_generic_T2_end(const void* src, size_t srcSize, void* dst, size_t dstCapacity, void* buff2)
{
ZSTD_outBuffer buffOut;
ZSTD_inBuffer buffIn;
(void)buff2;
ZSTD_CCtx_setParameter(g_cstream, ZSTD_p_compressionLevel, 1);
ZSTD_CCtx_setParameter(g_cstream, ZSTD_p_nbWorkers, 2);
buffOut.dst = dst;
buffOut.size = dstCapacity;
@@ -193,12 +241,11 @@ static size_t local_ZSTD_compress_generic_T2_end(void* dst, size_t dstCapacity,
return buffOut.pos;
}
static size_t local_ZSTD_compress_generic_T2_continue(void* dst, size_t dstCapacity, void* buff2, const void* src, size_t srcSize)
static size_t local_ZSTD_compress_generic_T2_continue(const void* src, size_t srcSize, void* dst, size_t dstCapacity, void* buff2)
{
ZSTD_outBuffer buffOut;
ZSTD_inBuffer buffIn;
(void)buff2;
ZSTD_CCtx_setParameter(g_cstream, ZSTD_p_compressionLevel, 1);
ZSTD_CCtx_setParameter(g_cstream, ZSTD_p_nbWorkers, 2);
buffOut.dst = dst;
buffOut.size = dstCapacity;
@@ -212,7 +259,7 @@ static size_t local_ZSTD_compress_generic_T2_continue(void* dst, size_t dstCapac
}
static ZSTD_DStream* g_dstream= NULL;
static size_t local_ZSTD_decompressStream(void* dst, size_t dstCapacity, void* buff2, const void* src, size_t srcSize)
static size_t local_ZSTD_decompressStream(const void* src, size_t srcSize, void* dst, size_t dstCapacity, void* buff2)
{
ZSTD_outBuffer buffOut;
ZSTD_inBuffer buffIn;
@@ -228,24 +275,28 @@ static size_t local_ZSTD_decompressStream(void* dst, size_t dstCapacity, void* b
return buffOut.pos;
}
static ZSTD_CCtx* g_zcc = NULL;
#ifndef ZSTD_DLL_IMPORT
size_t local_ZSTD_compressContinue(void* dst, size_t dstCapacity, void* buff2, const void* src, size_t srcSize)
size_t local_ZSTD_compressContinue(const void* src, size_t srcSize, void* dst, size_t dstCapacity, void* buff2)
{
(void)buff2;
ZSTD_compressBegin(g_zcc, 1 /* compressionLevel */);
ZSTD_parameters p;
ZSTD_frameParameters f = {1 /* contentSizeHeader*/, 0, 0};
p.fParams = f;
p.cParams = *(ZSTD_compressionParameters*)buff2;
ZSTD_compressBegin_advanced(g_zcc, NULL, 0, p, srcSize);
return ZSTD_compressEnd(g_zcc, dst, dstCapacity, src, srcSize);
}
#define FIRST_BLOCK_SIZE 8
size_t local_ZSTD_compressContinue_extDict(void* dst, size_t dstCapacity, void* buff2, const void* src, size_t srcSize)
size_t local_ZSTD_compressContinue_extDict(const void* src, size_t srcSize, void* dst, size_t dstCapacity, void* buff2)
{
BYTE firstBlockBuf[FIRST_BLOCK_SIZE];
(void)buff2;
ZSTD_parameters p;
ZSTD_frameParameters f = {1 , 0, 0};
p.fParams = f;
p.cParams = *(ZSTD_compressionParameters*)buff2;
ZSTD_compressBegin_advanced(g_zcc, NULL, 0, p, srcSize);
memcpy(firstBlockBuf, src, FIRST_BLOCK_SIZE);
ZSTD_compressBegin(g_zcc, 1);
{ size_t const compressResult = ZSTD_compressContinue(g_zcc, dst, dstCapacity, firstBlockBuf, FIRST_BLOCK_SIZE);
if (ZSTD_isError(compressResult)) { DISPLAY("local_ZSTD_compressContinue_extDict error : %s\n", ZSTD_getErrorName(compressResult)); return compressResult; }
@@ -255,7 +306,7 @@ size_t local_ZSTD_compressContinue_extDict(void* dst, size_t dstCapacity, void*
return ZSTD_compressEnd(g_zcc, dst, dstCapacity, (const BYTE*)src + FIRST_BLOCK_SIZE, srcSize - FIRST_BLOCK_SIZE);
}
size_t local_ZSTD_decompressContinue(void* dst, size_t dstCapacity, void* buff2, const void* src, size_t srcSize)
size_t local_ZSTD_decompressContinue(const void* src, size_t srcSize, void* dst, size_t dstCapacity, void* buff2)
{
size_t regeneratedSize = 0;
const BYTE* ip = (const BYTE*)buff2;
@@ -282,27 +333,28 @@ size_t local_ZSTD_decompressContinue(void* dst, size_t dstCapacity, void* buff2,
/*_*******************************************************
* Bench functions
*********************************************************/
static size_t benchMem(const void* src, size_t srcSize, U32 benchNb)
static size_t benchMem(const void* src, size_t srcSize, U32 benchNb, int cLevel, ZSTD_compressionParameters* cparams)
{
BYTE* dstBuff;
size_t const dstBuffSize = ZSTD_compressBound(srcSize);
void* buff2;
void* buff2, *buff1;
const char* benchName;
size_t (*benchFunction)(void* dst, size_t dstSize, void* verifBuff, const void* src, size_t srcSize);
double bestTime = 100000000.;
BMK_benchFn_t benchFunction;
BMK_customReturn_t r;
int errorcode = 0;
/* Selection */
switch(benchNb)
{
case 1:
benchFunction = local_ZSTD_compress; benchName = "compress(1)";
benchFunction = local_ZSTD_compress; benchName = "compress";
break;
case 2:
benchFunction = local_ZSTD_decompress; benchName = "decompress";
break;
#ifndef ZSTD_DLL_IMPORT
case 11:
benchFunction = local_ZSTD_compressContinue; benchName = "compressContinue(1)";
benchFunction = local_ZSTD_compressContinue; benchName = "compressContinue";
break;
case 12:
benchFunction = local_ZSTD_compressContinue_extDict; benchName = "compressContinue_extDict";
@@ -318,7 +370,7 @@ static size_t benchMem(const void* src, size_t srcSize, U32 benchNb)
break;
#endif
case 41:
benchFunction = local_ZSTD_compressStream; benchName = "compressStream(1)";
benchFunction = local_ZSTD_compressStream; benchName = "compressStream";
break;
case 42:
benchFunction = local_ZSTD_decompressStream; benchName = "decompressStream";
@@ -347,26 +399,59 @@ static size_t benchMem(const void* src, size_t srcSize, U32 benchNb)
free(dstBuff); free(buff2);
return 12;
}
buff1 = buff2;
if (g_zcc==NULL) g_zcc = ZSTD_createCCtx();
if (g_zdc==NULL) g_zdc = ZSTD_createDCtx();
if (g_cstream==NULL) g_cstream = ZSTD_createCStream();
if (g_dstream==NULL) g_dstream = ZSTD_createDStream();
/* DISPLAY("params: cLevel %d, wlog %d hlog %d clog %d slog %d slen %d tlen %d strat %d \n"
, cLevel, cparams->windowLog, cparams->hashLog, cparams->chainLog, cparams->searchLog,
cparams->searchLength, cparams->targetLength, cparams->strategy);*/
ZSTD_CCtx_setParameter(g_zcc, ZSTD_p_compressionLevel, cLevel);
ZSTD_CCtx_setParameter(g_zcc, ZSTD_p_windowLog, cparams->windowLog);
ZSTD_CCtx_setParameter(g_zcc, ZSTD_p_hashLog, cparams->hashLog);
ZSTD_CCtx_setParameter(g_zcc, ZSTD_p_chainLog, cparams->chainLog);
ZSTD_CCtx_setParameter(g_zcc, ZSTD_p_searchLog, cparams->searchLog);
ZSTD_CCtx_setParameter(g_zcc, ZSTD_p_minMatch, cparams->searchLength);
ZSTD_CCtx_setParameter(g_zcc, ZSTD_p_targetLength, cparams->targetLength);
ZSTD_CCtx_setParameter(g_zcc, ZSTD_p_compressionStrategy, cparams->strategy);
ZSTD_CCtx_setParameter(g_cstream, ZSTD_p_compressionLevel, cLevel);
ZSTD_CCtx_setParameter(g_cstream, ZSTD_p_windowLog, cparams->windowLog);
ZSTD_CCtx_setParameter(g_cstream, ZSTD_p_hashLog, cparams->hashLog);
ZSTD_CCtx_setParameter(g_cstream, ZSTD_p_chainLog, cparams->chainLog);
ZSTD_CCtx_setParameter(g_cstream, ZSTD_p_searchLog, cparams->searchLog);
ZSTD_CCtx_setParameter(g_cstream, ZSTD_p_minMatch, cparams->searchLength);
ZSTD_CCtx_setParameter(g_cstream, ZSTD_p_targetLength, cparams->targetLength);
ZSTD_CCtx_setParameter(g_cstream, ZSTD_p_compressionStrategy, cparams->strategy);
/* Preparation */
switch(benchNb)
{
case 1:
buff2 = (void*)cparams;
break;
case 2:
g_cSize = ZSTD_compress(buff2, dstBuffSize, src, srcSize, 1);
g_cSize = ZSTD_compress(buff2, dstBuffSize, src, srcSize, cLevel);
break;
#ifndef ZSTD_DLL_IMPORT
case 11:
buff2 = (void*)cparams;
break;
case 12:
buff2 = (void*)cparams;
break;
case 13 :
g_cSize = ZSTD_compress(buff2, dstBuffSize, src, srcSize, 1);
g_cSize = ZSTD_compress(buff2, dstBuffSize, src, srcSize, cLevel);
break;
case 31: /* ZSTD_decodeLiteralsBlock */
{ blockProperties_t bp;
ZSTD_frameHeader zfp;
size_t frameHeaderSize, skippedSize;
g_cSize = ZSTD_compress(dstBuff, dstBuffSize, src, srcSize, 1);
g_cSize = ZSTD_compress(dstBuff, dstBuffSize, src, srcSize, cLevel);
frameHeaderSize = ZSTD_getFrameHeader(&zfp, dstBuff, ZSTD_frameHeaderSize_min);
if (frameHeaderSize==0) frameHeaderSize = ZSTD_frameHeaderSize_min;
ZSTD_getcBlockSize(dstBuff+frameHeaderSize, dstBuffSize, &bp); /* Get 1st block type */
@@ -386,8 +471,8 @@ static size_t benchMem(const void* src, size_t srcSize, U32 benchNb)
const BYTE* ip = dstBuff;
const BYTE* iend;
size_t frameHeaderSize, cBlockSize;
ZSTD_compress(dstBuff, dstBuffSize, src, srcSize, 1); /* it would be better to use direct block compression here */
g_cSize = ZSTD_compress(dstBuff, dstBuffSize, src, srcSize, 1);
ZSTD_compress(dstBuff, dstBuffSize, src, srcSize, cLevel); /* it would be better to use direct block compression here */
g_cSize = ZSTD_compress(dstBuff, dstBuffSize, src, srcSize, cLevel);
frameHeaderSize = ZSTD_getFrameHeader(&zfp, dstBuff, ZSTD_frameHeaderSize_min);
if (frameHeaderSize==0) frameHeaderSize = ZSTD_frameHeaderSize_min;
ip += frameHeaderSize; /* Skip frame Header */
@@ -409,8 +494,11 @@ static size_t benchMem(const void* src, size_t srcSize, U32 benchNb)
case 31:
goto _cleanOut;
#endif
case 41 :
buff2 = (void*)cparams;
break;
case 42 :
g_cSize = ZSTD_compress(buff2, dstBuffSize, src, srcSize, 1);
g_cSize = ZSTD_compress(buff2, dstBuffSize, src, srcSize, cLevel);
break;
/* test functions */
@@ -419,58 +507,37 @@ static size_t benchMem(const void* src, size_t srcSize, U32 benchNb)
default : ;
}
/* warming up memory */
{ size_t i; for (i=0; i<dstBuffSize; i++) dstBuff[i]=(BYTE)i; }
/* benchmark loop */
{ U32 loopNb;
U32 nbRounds = (U32)((50 MB) / (srcSize+1)) + 1; /* initial conservative speed estimate */
# define TIME_SEC_MICROSEC (1*1000000ULL) /* 1 second */
# define TIME_SEC_NANOSEC (1*1000000000ULL) /* 1 second */
DISPLAY("%2i- %-30.30s : \r", benchNb, benchName);
for (loopNb = 1; loopNb <= g_nbIterations; loopNb++) {
UTIL_time_t clockStart;
size_t benchResult=0;
U32 roundNb;
UTIL_sleepMilli(5); /* give processor time to other processes */
UTIL_waitForNextTick();
clockStart = UTIL_getTime();
for (roundNb=0; roundNb < nbRounds; roundNb++) {
benchResult = benchFunction(dstBuff, dstBuffSize, buff2, src, srcSize);
if (ZSTD_isError(benchResult)) {
DISPLAY("ERROR ! %s() => %s !! \n", benchName, ZSTD_getErrorName(benchResult));
exit(1);
} }
{ U64 const clockSpanNano = UTIL_clockSpanNano(clockStart);
double const averageTime = (double)clockSpanNano / TIME_SEC_NANOSEC / nbRounds;
if (clockSpanNano > 0) {
if (averageTime < bestTime) bestTime = averageTime;
assert(bestTime > (1./2000000000));
nbRounds = (U32)(1. / bestTime); /* aim for 1 sec */
DISPLAY("%2i- %-30.30s : %7.1f MB/s (%9u)\r",
loopNb, benchName,
(double)srcSize / (1 MB) / bestTime,
(U32)benchResult);
} else {
assert(nbRounds < 40000000); /* avoid overflow */
nbRounds *= 100;
}
} } }
DISPLAY("%2u\n", benchNb);
{
r = BMK_benchFunction(benchFunction, buff2,
NULL, NULL, 1, &src, &srcSize,
(void * const * const)&dstBuff, &dstBuffSize, g_nbIterations);
if(r.error) {
DISPLAY("ERROR %d ! ! \n", r.error);
errorcode = r.error;
goto _cleanOut;
}
DISPLAY("%2u#Speed: %f MB/s - Size: %f MB - %s\n", benchNb, (double)srcSize / r.result.nanoSecPerRun * 1000, (double)r.result.sumOfReturn / 1000000, benchName);
}
_cleanOut:
free(buff1);
free(dstBuff);
free(buff2);
ZSTD_freeCCtx(g_zcc); g_zcc=NULL;
ZSTD_freeDCtx(g_zdc); g_zdc=NULL;
ZSTD_freeCStream(g_cstream); g_cstream=NULL;
ZSTD_freeDStream(g_dstream); g_dstream=NULL;
return 0;
return errorcode;
}
static int benchSample(U32 benchNb)
static int benchSample(U32 benchNb, int cLevel, ZSTD_compressionParameters* cparams)
{
size_t const benchedSize = g_sampleSize;
const char* name = "Sample 10MiB";
@@ -486,16 +553,16 @@ static int benchSample(U32 benchNb)
DISPLAY("\r%79s\r", "");
DISPLAY(" %s : \n", name);
if (benchNb)
benchMem(origBuff, benchedSize, benchNb);
benchMem(origBuff, benchedSize, benchNb, cLevel, cparams);
else
for (benchNb=0; benchNb<100; benchNb++) benchMem(origBuff, benchedSize, benchNb);
for (benchNb=0; benchNb<100; benchNb++) benchMem(origBuff, benchedSize, benchNb, cLevel, cparams);
free(origBuff);
return 0;
}
static int benchFiles(const char** fileNamesTable, const int nbFiles, U32 benchNb)
static int benchFiles(const char** fileNamesTable, const int nbFiles, U32 benchNb, int cLevel, ZSTD_compressionParameters* cparams)
{
/* Loop for each file */
int fileIdx;
@@ -540,9 +607,9 @@ static int benchFiles(const char** fileNamesTable, const int nbFiles, U32 benchN
DISPLAY("\r%79s\r", "");
DISPLAY(" %s : \n", inFileName);
if (benchNb)
benchMem(origBuff, benchedSize, benchNb);
benchMem(origBuff, benchedSize, benchNb, cLevel, cparams);
else
for (benchNb=0; benchNb<100; benchNb++) benchMem(origBuff, benchedSize, benchNb);
for (benchNb=0; benchNb<100; benchNb++) benchMem(origBuff, benchedSize, benchNb, cLevel, cparams);
free(origBuff);
}
@@ -567,6 +634,8 @@ static int usage_advanced(const char* exename)
DISPLAY( " -b# : test only function # \n");
DISPLAY( " -i# : iteration loops [1-9](default : %i)\n", NBLOOPS);
DISPLAY( " -P# : sample compressibility (default : %.1f%%)\n", COMPRESSIBILITY_DEFAULT * 100);
DISPLAY( " -l# : benchmark functions at that compression level (default : %i)\n", DEFAULT_CLEVEL);
DISPLAY( " --zstd : custom parameter selection. Format same as zstdcli \n");
return 0;
}
@@ -583,6 +652,8 @@ int main(int argc, const char** argv)
const char* exename = argv[0];
const char* input_filename = NULL;
U32 benchNb = 0, main_pause = 0;
int cLevel = DEFAULT_CLEVEL;
ZSTD_compressionParameters cparams = ZSTD_getCParams(cLevel, 0, 0);
DISPLAY(WELCOME_MESSAGE);
if (argc<1) return badusage(exename);
@@ -591,11 +662,29 @@ int main(int argc, const char** argv)
const char* argument = argv[i];
assert(argument != NULL);
/* Commands (note : aggregated commands are allowed) */
if (argument[0]=='-') {
if (longCommandWArg(&argument, "--zstd=")) {
for ( ; ;) {
if (longCommandWArg(&argument, "windowLog=") || longCommandWArg(&argument, "wlog=")) { cparams.windowLog = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; }
if (longCommandWArg(&argument, "chainLog=") || longCommandWArg(&argument, "clog=")) { cparams.chainLog = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; }
if (longCommandWArg(&argument, "hashLog=") || longCommandWArg(&argument, "hlog=")) { cparams.hashLog = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; }
if (longCommandWArg(&argument, "searchLog=") || longCommandWArg(&argument, "slog=")) { cparams.searchLog = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; }
if (longCommandWArg(&argument, "searchLength=") || longCommandWArg(&argument, "slen=")) { cparams.searchLength = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; }
if (longCommandWArg(&argument, "targetLength=") || longCommandWArg(&argument, "tlen=")) { cparams.targetLength = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; }
if (longCommandWArg(&argument, "strategy=") || longCommandWArg(&argument, "strat=")) { cparams.strategy = (ZSTD_strategy)(readU32FromChar(&argument)); if (argument[0]==',') { argument++; continue; } else break; }
if (longCommandWArg(&argument, "level=") || longCommandWArg(&argument, "lvl=")) { cLevel = (int)readU32FromChar(&argument); cparams = ZSTD_getCParams(cLevel, 0, 0); if (argument[0]==',') { argument++; continue; } else break; }
DISPLAY("invalid compression parameter \n");
return 1;
}
while (argument[1]!=0) {
argument++;
if (argument[0] != 0) {
DISPLAY("invalid --zstd= format\n");
return 1; // check the end of string
} else {
continue;
}
} else if (argument[0]=='-') { /* Commands (note : aggregated commands are allowed) */
argument++;
while (argument[0]!=0) {
switch(argument[0])
{
@@ -608,34 +697,34 @@ int main(int argc, const char** argv)
/* Select specific algorithm to bench */
case 'b':
benchNb = 0;
while ((argument[1]>= '0') && (argument[1]<= '9')) {
benchNb *= 10;
benchNb += argument[1] - '0';
{
argument++;
benchNb = readU32FromChar(&argument);
break;
}
break;
/* Modify Nb Iterations */
case 'i':
if ((argument[1] >='0') && (argument[1] <='9')) {
int iters = argument[1] - '0';
BMK_SetNbIterations(iters);
{
argument++;
BMK_SetNbIterations((int)readU32FromChar(&argument));
}
break;
/* Select compressibility of synthetic sample */
case 'P':
{ U32 proba32 = 0;
while ((argument[1]>= '0') && (argument[1]<= '9')) {
proba32 *= 10;
proba32 += argument[1] - '0';
argument++;
}
g_compressibility = (double)proba32 / 100.;
{ argument++;
g_compressibility = (double)readU32FromChar(&argument) / 100.;
}
break;
case 'l':
{ argument++;
cLevel = readU32FromChar(&argument);
cparams = ZSTD_getCParams(cLevel, 0, 0);
}
break;
/* Unknown command */
default : return badusage(exename);
@@ -648,10 +737,12 @@ int main(int argc, const char** argv)
if (!input_filename) { input_filename=argument; filenamesStart=i; continue; }
}
if (filenamesStart==0) /* no input file */
result = benchSample(benchNb);
result = benchSample(benchNb, cLevel, &cparams);
else
result = benchFiles(argv+filenamesStart, argc-filenamesStart, benchNb);
result = benchFiles(argv+filenamesStart, argc-filenamesStart, benchNb, cLevel, &cparams);
if (main_pause) { int unused; printf("press enter...\n"); unused = getchar(); (void)unused; }
tests/paramgrill.c
+1 -4
View File
@@ -162,18 +162,15 @@ const char* g_stratName[ZSTD_btultra+1] = {
"ZSTD_btlazy2 ", "ZSTD_btopt ", "ZSTD_btultra "};
/* TODO: support additional parameters (more files, fileSizes) */
//TODO: benchMem dctx can't = NULL in new system
static size_t
BMK_benchParam(BMK_result_t* resultPtr,
const void* srcBuffer, size_t srcSize,
ZSTD_CCtx* ctx, ZSTD_DCtx* dctx,
const ZSTD_compressionParameters cParams) {
BMK_return_t res = BMK_benchMem(srcBuffer,srcSize, &srcSize, 1, 0, &cParams, NULL, 0, ctx, dctx, 0, "File");
*resultPtr = res.result;
return res.errorCode;
return res.error;
}
static void BMK_printWinner(FILE* f, U32 cLevel, BMK_result_t result, ZSTD_compressionParameters params, size_t srcSize)
tests/playTests.sh
+29 -14
View File
@@ -48,6 +48,8 @@ fileRoundTripTest() {
$DIFF -q tmp.md5.1 tmp.md5.2
}
UNAME=$(uname)
isTerminal=false
if [ -t 0 ] && [ -t 1 ]
then
@@ -56,7 +58,10 @@ fi
isWindows=false
INTOVOID="/dev/null"
DEVDEVICE="/dev/random"
case "$UNAME" in
GNU) DEVDEVICE="/dev/random" ;;
*) DEVDEVICE="/dev/zero" ;;
esac
case "$OS" in
Windows*)
isWindows=true
@@ -65,7 +70,6 @@ case "$OS" in
;;
esac
UNAME=$(uname)
case "$UNAME" in
Darwin) MD5SUM="md5 -r" ;;
FreeBSD) MD5SUM="gmd5sum" ;;
@@ -408,7 +412,7 @@ $ECHO "\n===> cover dictionary builder : advanced options "
TESTFILE=../programs/zstdcli.c
./datagen > tmpDict
$ECHO "- Create first dictionary"
$ZSTD --train-cover=k=46,d=8 *.c ../programs/*.c -o tmpDict
$ZSTD --train-cover=k=46,d=8,split=80 *.c ../programs/*.c -o tmpDict
cp $TESTFILE tmp
$ZSTD -f tmp -D tmpDict
$ZSTD -d tmp.zst -D tmpDict -fo result
@@ -422,6 +426,11 @@ cmp tmpDict tmpDict1 && die "dictionaries should have different ID !"
$ECHO "- Create dictionary with size limit"
$ZSTD --train-cover=steps=8 *.c ../programs/*.c -o tmpDict2 --maxdict=4K
rm tmp*
$ECHO "- Compare size of dictionary from 90% training samples with 80% training samples"
$ZSTD --train-cover=split=90 -r *.c ../programs/*.c
$ZSTD --train-cover=split=80 -r *.c ../programs/*.c
$ECHO "- Create dictionary using all samples for both training and testing"
$ZSTD --train-cover=split=100 -r *.c ../programs/*.c
$ECHO "\n===> legacy dictionary builder "
@@ -544,16 +553,16 @@ $ZSTD --format=xz -V || LZMAMODE=0
if [ $LZMAMODE -eq 1 ]; then
$ECHO "xz support detected"
XZEXE=1
xz -V && lzma -V || XZEXE=0
xz -Q -V && lzma -Q -V || XZEXE=0
if [ $XZEXE -eq 1 ]; then
$ECHO "Testing zstd xz and lzma support"
./datagen > tmp
$ZSTD --format=lzma -f tmp
$ZSTD --format=xz -f tmp
xz -t -v tmp.xz
xz -t -v tmp.lzma
xz -f -k tmp
lzma -f -k --lzma1 tmp
xz -Q -t -v tmp.xz
xz -Q -t -v tmp.lzma
xz -Q -f -k tmp
lzma -Q -f -k --lzma1 tmp
$ZSTD -d -f -v tmp.xz
$ZSTD -d -f -v tmp.lzma
rm tmp*
@@ -565,13 +574,13 @@ if [ $LZMAMODE -eq 1 ]; then
$ECHO "Testing xz and lzma symlinks"
./datagen > tmp
./xz tmp
xz -d tmp.xz
xz -Q -d tmp.xz
./lzma tmp
lzma -d tmp.lzma
lzma -Q -d tmp.lzma
$ECHO "Testing unxz and unlzma symlinks"
xz tmp
xz -Q tmp
./xz -d tmp.xz
lzma tmp
lzma -Q tmp
./lzma -d tmp.lzma
rm xz unxz lzma unlzma
rm tmp*
@@ -731,8 +740,14 @@ $ECHO "\n===> zstd --list/-l error detection tests "
! $ZSTD -lv tmp1*
! $ZSTD --list -v tmp2 tmp12.zst
$ECHO "\n===> zstd --list/-l exits 1 when stdin is piped in"
! echo "piped STDIN" | $ZSTD --list
$ECHO "\n===> zstd --list/-l errors when presented with stdin / no files"
! $ZSTD -l
! $ZSTD -l -
! $ZSTD -l < tmp1.zst
! $ZSTD -l - < tmp1.zst
! $ZSTD -l - tmp1.zst
! $ZSTD -l - tmp1.zst < tmp1.zst
$ZSTD -l tmp1.zst < tmp1.zst # but doesn't error just because stdin is not a tty
$ECHO "\n===> zstd --list/-l test with null files "
./datagen -g0 > tmp5