ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'dev' into adapt

Browse Source
This commit is contained in:
Yann Collet
2018-09-19 12:43:42 -07:00
parent c71c4f23d7 c8ff5200f9
commit 2f78228f65
78 changed files with 4962 additions and 1660 deletions
Download Patch File Download Diff File Expand all files Collapse all files
programs/Makefile
+11 -1
View File
@@ -132,6 +132,15 @@ else
LZ4_MSG := $(NO_LZ4_MSG)
endif
# enable backtrace symbol names for Linux/Darwin
ALL_SYMBOLS := 0
ifeq (,$(filter Windows%, $(OS)))
ifeq ($(ALL_SYMBOLS), 1)
DEBUGFLAGS_LD+=-rdynamic
endif
endif
.PHONY: default
default: zstd-release
@@ -144,7 +153,7 @@ allVariants: zstd zstd-compress zstd-decompress zstd-small zstd-nolegacy
$(ZSTDDECOMP_O): CFLAGS += $(ALIGN_LOOP)
zstd : CPPFLAGS += $(THREAD_CPP) $(ZLIBCPP) $(LZMACPP) $(LZ4CPP)
zstd : LDFLAGS += $(THREAD_LD) $(ZLIBLD) $(LZMALD) $(LZ4LD)
zstd : LDFLAGS += $(THREAD_LD) $(ZLIBLD) $(LZMALD) $(LZ4LD) $(DEBUGFLAGS_LD)
zstd : CPPFLAGS += -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT)
zstd : $(ZSTDLIB_FILES) zstdcli.o fileio.o bench.o datagen.o dibio.o
@echo "$(THREAD_MSG)"
@@ -158,6 +167,7 @@ endif
.PHONY: zstd-release
zstd-release: DEBUGFLAGS :=
zstd-release: DEBUGFLAGS_LD :=
zstd-release: zstd
zstd32 : CPPFLAGS += $(THREAD_CPP)
programs/README.md
+8
View File
@@ -61,6 +61,13 @@ There are however other Makefile targets that create different variations of CLI
In which case, linking stage will fail if `lz4` library cannot be found.
This is useful to prevent silent feature disabling.
- __ALL_SYMBOLS__ : `zstd` can display a stack backtrace if the execution
generates a runtime exception. By default, this feature may be
degraded/disabled on some platforms unless additional compiler directives are
applied. When triaging a runtime issue, enabling this feature can provided
more context to determine the location of the fault.
Example : `make zstd ALL_SYMBOLS=1`
#### Aggregation of parameters
CLI supports aggregation of parameters i.e. `-b1`, `-e18`, and `-i1` can be joined into `-b1e18i1`.
@@ -151,6 +158,7 @@ Advanced arguments :
Dictionary builder :
--train ## : create a dictionary from a training set of files
--train-cover[=k=#,d=#,steps=#,split=#] : use the cover algorithm with optional args
--train-fastcover[=k=#,d=#,f=#,steps=#,split=#,accel=#] : use the fastcover algorithm with optional args
--train-legacy[=s=#] : use the legacy algorithm with selectivity (default: 9)
-o file : `file` is dictionary name (default: dictionary)
--maxdict=# : limit dictionary to specified size (default: 112640)
programs/bench.c
+530 -462
View File
@@ -63,7 +63,12 @@
#define MB *(1 <<20)
#define GB *(1U<<30)
static const size_t maxMemory = (sizeof(size_t)==4) ? (2 GB - 64 MB) : (size_t)(1ULL << ((sizeof(size_t)*8)-31));
#define BMK_RUNTEST_DEFAULT_MS 1000
static const size_t maxMemory = (sizeof(size_t)==4) ?
/* 32-bit */ (2 GB - 64 MB) :
/* 64-bit */ (size_t)(1ULL << ((sizeof(size_t)*8)-31));
/* *************************************
* console display
@@ -97,26 +102,26 @@ static UTIL_time_t g_displayClock = UTIL_TIME_INITIALIZER;
return errorNum; \
}
#define EXM_THROW(errorNum, retType, ...) { \
#define RETURN_ERROR(errorNum, retType, ...) { \
retType r; \
memset(&r, 0, sizeof(retType)); \
DEBUGOUTPUT("%s: %i: \n", __FILE__, __LINE__); \
DISPLAYLEVEL(1, "Error %i : ", errorNum); \
DISPLAYLEVEL(1, __VA_ARGS__); \
DISPLAYLEVEL(1, " \n"); \
r.error = errorNum; \
r.tag = errorNum; \
return r; \
}
/* error without displaying */
#define EXM_THROW_ND(errorNum, retType, ...) { \
#define RETURN_QUIET_ERROR(errorNum, retType, ...) { \
retType r; \
memset(&r, 0, sizeof(retType)); \
DEBUGOUTPUT("%s: %i: \n", __FILE__, __LINE__); \
DEBUGOUTPUT("Error %i : ", errorNum); \
DEBUGOUTPUT(__VA_ARGS__); \
DEBUGOUTPUT(" \n"); \
r.error = errorNum; \
r.tag = errorNum; \
return r; \
}
@@ -124,16 +129,15 @@ static UTIL_time_t g_displayClock = UTIL_TIME_INITIALIZER;
* Benchmark Parameters
***************************************/
BMK_advancedParams_t BMK_initAdvancedParams(void) {
BMK_advancedParams_t res = {
BMK_advancedParams_t BMK_initAdvancedParams(void) {
BMK_advancedParams_t const res = {
BMK_both, /* mode */
BMK_timeMode, /* loopMode */
BMK_TIMETEST_DEFAULT_S, /* nbSeconds */
0, /* blockSize */
0, /* nbWorkers */
0, /* realTime */
0, /* additionalParam */
0, /* ldmFlag */
0, /* ldmFlag */
0, /* ldmMinMatch */
0, /* ldmHashLog */
0, /* ldmBuckSizeLog */
@@ -156,20 +160,13 @@ typedef struct {
size_t resSize;
} blockParam_t;
struct BMK_timeState_t{
unsigned nbLoops;
U64 timeRemaining;
UTIL_time_t coolTime;
U64 fastestTime;
};
#undef MIN
#undef MAX
#define MIN(a,b) ((a) < (b) ? (a) : (b))
#define MAX(a,b) ((a) > (b) ? (a) : (b))
static void BMK_initCCtx(ZSTD_CCtx* ctx,
const void* dictBuffer, size_t dictBufferSize, int cLevel,
static void BMK_initCCtx(ZSTD_CCtx* ctx,
const void* dictBuffer, size_t dictBufferSize, int cLevel,
const ZSTD_compressionParameters* comprParams, const BMK_advancedParams_t* adv) {
ZSTD_CCtx_reset(ctx);
ZSTD_CCtx_resetParameters(ctx);
@@ -194,15 +191,15 @@ static void BMK_initCCtx(ZSTD_CCtx* ctx,
ZSTD_CCtx_loadDictionary(ctx, dictBuffer, dictBufferSize);
}
static void BMK_initDCtx(ZSTD_DCtx* dctx,
static void BMK_initDCtx(ZSTD_DCtx* dctx,
const void* dictBuffer, size_t dictBufferSize) {
ZSTD_DCtx_reset(dctx);
ZSTD_DCtx_loadDictionary(dctx, dictBuffer, dictBufferSize);
}
typedef struct {
ZSTD_CCtx* ctx;
ZSTD_CCtx* cctx;
const void* dictBuffer;
size_t dictBufferSize;
int cLevel;
@@ -212,7 +209,7 @@ typedef struct {
static size_t local_initCCtx(void* payload) {
BMK_initCCtxArgs* ag = (BMK_initCCtxArgs*)payload;
BMK_initCCtx(ag->ctx, ag->dictBuffer, ag->dictBufferSize, ag->cLevel, ag->comprParams, ag->adv);
BMK_initCCtx(ag->cctx, ag->dictBuffer, ag->dictBufferSize, ag->cLevel, ag->comprParams, ag->adv);
return 0;
}
@@ -228,26 +225,24 @@ static size_t local_initDCtx(void* payload) {
return 0;
}
/* additional argument is just the context */
/* `addArgs` is the context */
static size_t local_defaultCompress(
const void* srcBuffer, size_t srcSize,
void* dstBuffer, size_t dstSize,
void* addArgs) {
const void* srcBuffer, size_t srcSize,
void* dstBuffer, size_t dstSize,
void* addArgs)
{
size_t moreToFlush = 1;
ZSTD_CCtx* ctx = (ZSTD_CCtx*)addArgs;
ZSTD_CCtx* const cctx = (ZSTD_CCtx*)addArgs;
ZSTD_inBuffer in;
ZSTD_outBuffer out;
in.src = srcBuffer;
in.size = srcSize;
in.pos = 0;
out.dst = dstBuffer;
out.size = dstSize;
out.pos = 0;
in.src = srcBuffer; in.size = srcSize; in.pos = 0;
out.dst = dstBuffer; out.size = dstSize; out.pos = 0;
while (moreToFlush) {
if(out.pos == out.size) {
return (size_t)-ZSTD_error_dstSize_tooSmall;
}
moreToFlush = ZSTD_compress_generic(ctx, &out, &in, ZSTD_e_end);
moreToFlush = ZSTD_compress_generic(cctx, &out, &in, ZSTD_e_end);
if (ZSTD_isError(moreToFlush)) {
return moreToFlush;
}
@@ -255,27 +250,23 @@ static size_t local_defaultCompress(
return out.pos;
}
/* additional argument is just the context */
/* `addArgs` is the context */
static size_t local_defaultDecompress(
const void* srcBuffer, size_t srcSize,
void* dstBuffer, size_t dstSize,
void* addArgs) {
const void* srcBuffer, size_t srcSize,
void* dstBuffer, size_t dstCapacity,
void* addArgs)
{
size_t moreToFlush = 1;
ZSTD_DCtx* dctx = (ZSTD_DCtx*)addArgs;
ZSTD_DCtx* const dctx = (ZSTD_DCtx*)addArgs;
ZSTD_inBuffer in;
ZSTD_outBuffer out;
in.src = srcBuffer;
in.size = srcSize;
in.pos = 0;
out.dst = dstBuffer;
out.size = dstSize;
out.pos = 0;
in.src = srcBuffer; in.size = srcSize; in.pos = 0;
out.dst = dstBuffer; out.size = dstCapacity; out.pos = 0;
while (moreToFlush) {
if(out.pos == out.size) {
return (size_t)-ZSTD_error_dstSize_tooSmall;
}
moreToFlush = ZSTD_decompress_generic(dctx,
&out, &in);
moreToFlush = ZSTD_decompress_generic(dctx, &out, &in);
if (ZSTD_isError(moreToFlush)) {
return moreToFlush;
}
@@ -284,198 +275,307 @@ static size_t local_defaultDecompress(
}
/* initFn will be measured once, bench fn will be measured x times */
/* benchFn should return error value or out Size */
/* takes # of blocks and list of size & stuff for each. */
/* only does looping */
/* note time per loop could be zero if interval too short */
BMK_customReturn_t BMK_benchFunction(
BMK_benchFn_t benchFn, void* benchPayload,
BMK_initFn_t initFn, void* initPayload,
size_t blockCount,
const void* const * const srcBlockBuffers, const size_t* srcBlockSizes,
void* const * const dstBlockBuffers, const size_t* dstBlockCapacities, size_t* blockResult,
unsigned nbLoops) {
size_t dstSize = 0;
U64 totalTime;
BMK_customReturn_t retval;
UTIL_time_t clockStart;
/*=== Benchmarking an arbitrary function ===*/
int BMK_isSuccessful_runOutcome(BMK_runOutcome_t outcome)
{
return outcome.tag == 0;
}
/* warning : this function will stop program execution if outcome is invalid !
* check outcome validity first, using BMK_isValid_runResult() */
BMK_runTime_t BMK_extract_runTime(BMK_runOutcome_t outcome)
{
assert(outcome.tag == 0);
return outcome.internal_never_use_directly;
}
static BMK_runOutcome_t BMK_runOutcome_error(void)
{
BMK_runOutcome_t b;
memset(&b, 0, sizeof(b));
b.tag = 1;
return b;
}
static BMK_runOutcome_t BMK_setValid_runTime(BMK_runTime_t runTime)
{
BMK_runOutcome_t outcome;
outcome.tag = 0;
outcome.internal_never_use_directly = runTime;
return outcome;
}
/* initFn will be measured once, benchFn will be measured `nbLoops` times */
/* initFn is optional, provide NULL if none */
/* benchFn must return size_t field compliant with ZSTD_isError for error valuee */
/* takes # of blocks and list of size & stuff for each. */
/* can report result of benchFn for each block into blockResult. */
/* blockResult is optional, provide NULL if this information is not required */
/* note : time per loop could be zero if run time < timer resolution */
BMK_runOutcome_t BMK_benchFunction(
BMK_benchFn_t benchFn, void* benchPayload,
BMK_initFn_t initFn, void* initPayload,
size_t blockCount,
const void* const * srcBlockBuffers, const size_t* srcBlockSizes,
void* const * dstBlockBuffers, const size_t* dstBlockCapacities,
size_t* blockResults,
unsigned nbLoops)
{
size_t dstSize = 0;
if(!nbLoops) {
EXM_THROW_ND(1, BMK_customReturn_t, "nbLoops must be nonzero \n");
RETURN_QUIET_ERROR(2, BMK_runOutcome_t, "nbLoops must be nonzero ");
}
{
size_t i;
/* init */
{ size_t i;
for(i = 0; i < blockCount; i++) {
memset(dstBlockBuffers[i], 0xE5, dstBlockCapacities[i]); /* warm up and erase result buffer */
}
#if 0
/* based on testing these seem to lower accuracy of multiple calls of 1 nbLoops vs 1 call of multiple nbLoops
* (Makes former slower)
/* based on testing these seem to lower accuracy of multiple calls of 1 nbLoops vs 1 call of multiple nbLoops
* (Makes former slower)
*/
UTIL_sleepMilli(5); /* give processor time to other processes */
UTIL_waitForNextTick();
#endif
}
{
unsigned i, j;
clockStart = UTIL_getTime();
if(initFn != NULL) { initFn(initPayload); }
for(i = 0; i < nbLoops; i++) {
for(j = 0; j < blockCount; j++) {
size_t res = benchFn(srcBlockBuffers[j], srcBlockSizes[j], dstBlockBuffers[j], dstBlockCapacities[j], benchPayload);
/* benchmark */
{ UTIL_time_t const clockStart = UTIL_getTime();
unsigned loopNb, blockNb;
if (initFn != NULL) initFn(initPayload);
for (loopNb = 0; loopNb < nbLoops; loopNb++) {
for (blockNb = 0; blockNb < blockCount; blockNb++) {
size_t const res = benchFn(srcBlockBuffers[blockNb], srcBlockSizes[blockNb],
dstBlockBuffers[blockNb], dstBlockCapacities[blockNb],
benchPayload);
if(ZSTD_isError(res)) {
EXM_THROW_ND(2, BMK_customReturn_t, "Function benchmarking failed on block %u of size %u : %s \n",
j, (U32)dstBlockCapacities[j], ZSTD_getErrorName(res));
} else if(i == nbLoops - 1) {
RETURN_QUIET_ERROR(2, BMK_runOutcome_t,
"Function benchmark failed on block %u of size %u : %s",
blockNb, (U32)dstBlockCapacities[blockNb], ZSTD_getErrorName(res));
} else if (loopNb == 0) {
dstSize += res;
if(blockResult != NULL) {
blockResult[j] = res;
}
}
}
}
totalTime = UTIL_clockSpanNano(clockStart);
}
if (blockResults != NULL) blockResults[blockNb] = res;
} }
} /* for (loopNb = 0; loopNb < nbLoops; loopNb++) */
retval.error = 0;
retval.result.nanoSecPerRun = totalTime / nbLoops;
retval.result.sumOfReturn = dstSize;
return retval;
}
#define MINUSABLETIME 500000000ULL /* 0.5 seconds in ns */
void BMK_resetTimeState(BMK_timedFnState_t* r, unsigned nbSeconds) {
r->nbLoops = 1;
r->timeRemaining = (U64)nbSeconds * TIMELOOP_NANOSEC;
r->coolTime = UTIL_getTime();
r->fastestTime = (U64)(-1LL);
{ U64 const totalTime = UTIL_clockSpanNano(clockStart);
BMK_runTime_t rt;
rt.nanoSecPerRun = totalTime / nbLoops;
rt.sumOfReturn = dstSize;
return BMK_setValid_runTime(rt);
} }
}
BMK_timedFnState_t* BMK_createTimeState(unsigned nbSeconds) {
BMK_timedFnState_t* r = (BMK_timedFnState_t*)malloc(sizeof(struct BMK_timeState_t));
if(r == NULL) {
return r;
}
BMK_resetTimeState(r, nbSeconds);
/* ==== Benchmarking any function, providing intermediate results ==== */
struct BMK_timedFnState_s {
U64 timeSpent_ns;
U64 timeBudget_ns;
U64 runBudget_ns;
BMK_runTime_t fastestRun;
unsigned nbLoops;
UTIL_time_t coolTime;
}; /* typedef'd to BMK_timedFnState_t within bench.h */
BMK_timedFnState_t* BMK_createTimedFnState(unsigned total_ms, unsigned run_ms)
{
BMK_timedFnState_t* const r = (BMK_timedFnState_t*)malloc(sizeof(*r));
if (r == NULL) return NULL; /* malloc() error */
BMK_resetTimedFnState(r, total_ms, run_ms);
return r;
}
void BMK_freeTimeState(BMK_timedFnState_t* state) {
void BMK_freeTimedFnState(BMK_timedFnState_t* state) {
free(state);
}
/* make option for dstBlocks to be */
BMK_customTimedReturn_t BMK_benchFunctionTimed(
BMK_timedFnState_t* cont,
BMK_benchFn_t benchFn, void* benchPayload,
BMK_initFn_t initFn, void* initPayload,
size_t blockCount,
const void* const* const srcBlockBuffers, const size_t* srcBlockSizes,
void * const * const dstBlockBuffers, const size_t * dstBlockCapacities, size_t* blockResults)
void BMK_resetTimedFnState(BMK_timedFnState_t* timedFnState, unsigned total_ms, unsigned run_ms)
{
U64 fastest = cont->fastestTime;
if (!total_ms) total_ms = 1 ;
if (!run_ms) run_ms = 1;
if (run_ms > total_ms) run_ms = total_ms;
timedFnState->timeSpent_ns = 0;
timedFnState->timeBudget_ns = (U64)total_ms * TIMELOOP_NANOSEC / 1000;
timedFnState->runBudget_ns = (U64)run_ms * TIMELOOP_NANOSEC / 1000;
timedFnState->fastestRun.nanoSecPerRun = (U64)(-1LL);
timedFnState->fastestRun.sumOfReturn = (size_t)(-1LL);
timedFnState->nbLoops = 1;
timedFnState->coolTime = UTIL_getTime();
}
/* Tells if nb of seconds set in timedFnState for all runs is spent.
* note : this function will return 1 if BMK_benchFunctionTimed() has actually errored. */
int BMK_isCompleted_TimedFn(const BMK_timedFnState_t* timedFnState)
{
return (timedFnState->timeSpent_ns >= timedFnState->timeBudget_ns);
}
#define MINUSABLETIME (TIMELOOP_NANOSEC / 2) /* 0.5 seconds */
BMK_runOutcome_t BMK_benchTimedFn(
BMK_timedFnState_t* cont,
BMK_benchFn_t benchFn, void* benchPayload,
BMK_initFn_t initFn, void* initPayload,
size_t blockCount,
const void* const* srcBlockBuffers, const size_t* srcBlockSizes,
void * const * dstBlockBuffers, const size_t * dstBlockCapacities,
size_t* blockResults)
{
U64 const runBudget_ns = cont->runBudget_ns;
U64 const runTimeMin_ns = runBudget_ns / 2;
int completed = 0;
BMK_customTimedReturn_t r;
r.completed = 0;
BMK_runTime_t bestRunTime = cont->fastestRun;
while (!completed) {
BMK_runOutcome_t runResult;
while(!r.completed && !completed)
{
/* Overheat protection */
if (UTIL_clockSpanMicro(cont->coolTime) > ACTIVEPERIOD_MICROSEC) {
DEBUGOUTPUT("\rcooling down ... \r");
UTIL_sleep(COOLPERIOD_SEC);
cont->coolTime = UTIL_getTime();
}
/* reinitialize capacity */
r.result = BMK_benchFunction(benchFn, benchPayload, initFn, initPayload,
blockCount, srcBlockBuffers, srcBlockSizes, dstBlockBuffers, dstBlockCapacities, blockResults, cont->nbLoops);
if(r.result.error) { /* completed w/ error */
r.completed = 1;
return r;
runResult = BMK_benchFunction(benchFn, benchPayload,
initFn, initPayload,
blockCount,
srcBlockBuffers, srcBlockSizes,
dstBlockBuffers, dstBlockCapacities,
blockResults,
cont->nbLoops);
if(!BMK_isSuccessful_runOutcome(runResult)) { /* error : move out */
return BMK_runOutcome_error();
}
{ U64 const loopDuration = r.result.result.nanoSecPerRun * cont->nbLoops;
r.completed = (cont->timeRemaining <= loopDuration);
cont->timeRemaining -= loopDuration;
if (loopDuration > (TIMELOOP_NANOSEC / 100)) {
fastest = MIN(fastest, r.result.result.nanoSecPerRun);
if(loopDuration >= MINUSABLETIME) {
r.result.result.nanoSecPerRun = fastest;
cont->fastestTime = fastest;
}
cont->nbLoops = (U32)(TIMELOOP_NANOSEC / r.result.result.nanoSecPerRun) + 1;
{ BMK_runTime_t const newRunTime = BMK_extract_runTime(runResult);
U64 const loopDuration_ns = newRunTime.nanoSecPerRun * cont->nbLoops;
cont->timeSpent_ns += loopDuration_ns;
/* estimate nbLoops for next run to last approximately 1 second */
if (loopDuration_ns > (runBudget_ns / 50)) {
U64 const fastestRun_ns = MIN(bestRunTime.nanoSecPerRun, newRunTime.nanoSecPerRun);
cont->nbLoops = (U32)(runBudget_ns / fastestRun_ns) + 1;
} else {
const unsigned multiplier = 2;
/* previous run was too short : blindly increase workload by x multiplier */
const unsigned multiplier = 10;
assert(cont->nbLoops < ((unsigned)-1) / multiplier); /* avoid overflow */
cont->nbLoops *= multiplier;
}
if(loopDuration < MINUSABLETIME) { /* don't report results which have time too low */
continue;
}
if(loopDuration_ns < runTimeMin_ns) {
/* don't report results for which benchmark run time was too small : increased risks of rounding errors */
assert(completed == 0);
continue;
} else {
if(newRunTime.nanoSecPerRun < bestRunTime.nanoSecPerRun) {
bestRunTime = newRunTime;
}
completed = 1;
}
}
completed = 1;
}
return r;
} /* while (!completed) */
return BMK_setValid_runTime(bestRunTime);
}
/* benchMem with no allocation */
static BMK_return_t BMK_benchMemAdvancedNoAlloc(
const void ** const srcPtrs, size_t* const srcSizes,
void** const cPtrs, size_t* const cCapacities, size_t* const cSizes,
void** const resPtrs, size_t* const resSizes,
void** resultBufferPtr, void* compressedBuffer,
const size_t maxCompressedSize,
BMK_timedFnState_t* timeStateCompress, BMK_timedFnState_t* timeStateDecompress,
const void* srcBuffer, size_t srcSize,
const size_t* fileSizes, unsigned nbFiles,
const int cLevel, const ZSTD_compressionParameters* comprParams,
const void* dictBuffer, size_t dictBufferSize,
ZSTD_CCtx* ctx, ZSTD_DCtx* dctx,
int displayLevel, const char* displayName, const BMK_advancedParams_t* adv)
/* ================================================================= */
/* Benchmark Zstandard, mem-to-mem scenarios */
/* ================================================================= */
int BMK_isSuccessful_benchOutcome(BMK_benchOutcome_t outcome)
{
size_t const blockSize = ((adv->blockSize>=32 && (adv->mode != BMK_decodeOnly)) ? adv->blockSize : srcSize) + (!srcSize); /* avoid div by 0 */
BMK_return_t results = { { 0, 0, 0, 0 }, 0 } ;
return outcome.tag == 0;
}
BMK_benchResult_t BMK_extract_benchResult(BMK_benchOutcome_t outcome)
{
assert(outcome.tag == 0);
return outcome.internal_never_use_directly;
}
static BMK_benchOutcome_t BMK_benchOutcome_error(void)
{
BMK_benchOutcome_t b;
memset(&b, 0, sizeof(b));
b.tag = 1;
return b;
}
static BMK_benchOutcome_t BMK_benchOutcome_setValidResult(BMK_benchResult_t result)
{
BMK_benchOutcome_t b;
b.tag = 0;
b.internal_never_use_directly = result;
return b;
}
/* benchMem with no allocation */
static BMK_benchOutcome_t BMK_benchMemAdvancedNoAlloc(
const void** srcPtrs, size_t* srcSizes,
void** cPtrs, size_t* cCapacities, size_t* cSizes,
void** resPtrs, size_t* resSizes,
void** resultBufferPtr, void* compressedBuffer,
size_t maxCompressedSize,
BMK_timedFnState_t* timeStateCompress,
BMK_timedFnState_t* timeStateDecompress,
const void* srcBuffer, size_t srcSize,
const size_t* fileSizes, unsigned nbFiles,
const int cLevel, const ZSTD_compressionParameters* comprParams,
const void* dictBuffer, size_t dictBufferSize,
ZSTD_CCtx* cctx, ZSTD_DCtx* dctx,
int displayLevel, const char* displayName,
const BMK_advancedParams_t* adv)
{
size_t const blockSize = ((adv->blockSize>=32 && (adv->mode != BMK_decodeOnly)) ? adv->blockSize : srcSize) + (!srcSize); /* avoid div by 0 */
BMK_benchResult_t benchResult;
size_t const loadedCompressedSize = srcSize;
size_t cSize = 0;
double ratio = 0.;
U32 nbBlocks;
if(!ctx || !dctx)
EXM_THROW(31, BMK_return_t, "error: passed in null context");
assert(cctx != NULL); assert(dctx != NULL);
/* init */
if (strlen(displayName)>17) displayName += strlen(displayName)-17; /* display last 17 characters */
memset(&benchResult, 0, sizeof(benchResult));
if (strlen(displayName)>17) displayName += strlen(displayName) - 17; /* display last 17 characters */
if (adv->mode == BMK_decodeOnly) { /* benchmark only decompression : source must be already compressed */
const char* srcPtr = (const char*)srcBuffer;
U64 totalDSize64 = 0;
U32 fileNb;
for (fileNb=0; fileNb<nbFiles; fileNb++) {
U64 const fSize64 = ZSTD_findDecompressedSize(srcPtr, fileSizes[fileNb]);
if (fSize64==0) EXM_THROW(32, BMK_return_t, "Impossible to determine original size ");
if (fSize64==0) RETURN_ERROR(32, BMK_benchOutcome_t, "Impossible to determine original size ");
totalDSize64 += fSize64;
srcPtr += fileSizes[fileNb];
}
{ size_t const decodedSize = (size_t)totalDSize64;
assert((U64)decodedSize == totalDSize64); /* check overflow */
free(*resultBufferPtr);
*resultBufferPtr = malloc(decodedSize);
if (!(*resultBufferPtr)) {
EXM_THROW(33, BMK_return_t, "not enough memory");
RETURN_ERROR(33, BMK_benchOutcome_t, "not enough memory");
}
if (totalDSize64 > decodedSize) {
free(*resultBufferPtr);
EXM_THROW(32, BMK_return_t, "original size is too large"); /* size_t overflow */
if (totalDSize64 > decodedSize) { /* size_t overflow */
free(*resultBufferPtr);
RETURN_ERROR(32, BMK_benchOutcome_t, "original size is too large");
}
cSize = srcSize;
srcSize = decodedSize;
ratio = (double)srcSize / (double)cSize;
}
}
}
/* Init data blocks */
@@ -489,21 +589,21 @@ static BMK_return_t BMK_benchMemAdvancedNoAlloc(
U32 const blockEnd = nbBlocks + nbBlocksforThisFile;
for ( ; nbBlocks<blockEnd; nbBlocks++) {
size_t const thisBlockSize = MIN(remaining, blockSize);
srcPtrs[nbBlocks] = (const void*)srcPtr;
srcPtrs[nbBlocks] = srcPtr;
srcSizes[nbBlocks] = thisBlockSize;
cPtrs[nbBlocks] = (void*)cPtr;
cPtrs[nbBlocks] = cPtr;
cCapacities[nbBlocks] = (adv->mode == BMK_decodeOnly) ? thisBlockSize : ZSTD_compressBound(thisBlockSize);
resPtrs[nbBlocks] = (void*)resPtr;
resPtrs[nbBlocks] = resPtr;
resSizes[nbBlocks] = (adv->mode == BMK_decodeOnly) ? (size_t) ZSTD_findDecompressedSize(srcPtr, thisBlockSize) : thisBlockSize;
srcPtr += thisBlockSize;
cPtr += cCapacities[nbBlocks];
resPtr += thisBlockSize;
remaining -= thisBlockSize;
}
}
}
}
}
/* warmimg up memory */
/* warmimg up `compressedBuffer` */
if (adv->mode == BMK_decodeOnly) {
memcpy(compressedBuffer, srcBuffer, loadedCompressedSize);
} else {
@@ -511,151 +611,105 @@ static BMK_return_t BMK_benchMemAdvancedNoAlloc(
}
/* Bench */
{
U64 const crcOrig = (adv->mode == BMK_decodeOnly) ? 0 : XXH64(srcBuffer, srcSize, 0);
{ U64 const crcOrig = (adv->mode == BMK_decodeOnly) ? 0 : XXH64(srcBuffer, srcSize, 0);
# define NB_MARKS 4
const char* const marks[NB_MARKS] = { " |", " /", " =", "\\" };
const char* marks[NB_MARKS] = { " |", " /", " =", " \\" };
U32 markNb = 0;
DISPLAYLEVEL(2, "\r%79s\r", "");
int compressionCompleted = (adv->mode == BMK_decodeOnly);
int decompressionCompleted = (adv->mode == BMK_compressOnly);
BMK_initCCtxArgs cctxprep;
BMK_initDCtxArgs dctxprep;
cctxprep.cctx = cctx;
cctxprep.dictBuffer = dictBuffer;
cctxprep.dictBufferSize = dictBufferSize;
cctxprep.cLevel = cLevel;
cctxprep.comprParams = comprParams;
cctxprep.adv = adv;
dctxprep.dctx = dctx;
dctxprep.dictBuffer = dictBuffer;
dctxprep.dictBufferSize = dictBufferSize;
DISPLAYLEVEL(2, "\r%70s\r", ""); /* blank line */
DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->\r", marks[markNb], displayName, (U32)srcSize);
{
BMK_initCCtxArgs cctxprep;
BMK_initDCtxArgs dctxprep;
cctxprep.ctx = ctx;
cctxprep.dictBuffer = dictBuffer;
cctxprep.dictBufferSize = dictBufferSize;
cctxprep.cLevel = cLevel;
cctxprep.comprParams = comprParams;
cctxprep.adv = adv;
dctxprep.dctx = dctx;
dctxprep.dictBuffer = dictBuffer;
dctxprep.dictBufferSize = dictBufferSize;
if(adv->loopMode == BMK_timeMode) {
BMK_customTimedReturn_t intermediateResultCompress;
BMK_customTimedReturn_t intermediateResultDecompress;
if(adv->mode == BMK_compressOnly) {
intermediateResultCompress.completed = 0;
intermediateResultDecompress.completed = 1;
} else if (adv->mode == BMK_decodeOnly) {
intermediateResultCompress.completed = 1;
intermediateResultDecompress.completed = 0;
} else { /* both */
intermediateResultCompress.completed = 0;
intermediateResultDecompress.completed = 0;
}
while(!(intermediateResultCompress.completed && intermediateResultDecompress.completed)) {
if(!intermediateResultCompress.completed) {
intermediateResultCompress = BMK_benchFunctionTimed(timeStateCompress, &local_defaultCompress, (void*)ctx, &local_initCCtx, (void*)&cctxprep,
nbBlocks, srcPtrs, srcSizes, cPtrs, cCapacities, cSizes);
if(intermediateResultCompress.result.error) {
results.error = intermediateResultCompress.result.error;
return results;
}
ratio = (double)(srcSize / intermediateResultCompress.result.result.sumOfReturn);
{
int const ratioAccuracy = (ratio < 10.) ? 3 : 2;
results.result.cSpeed = (srcSize * TIMELOOP_NANOSEC / intermediateResultCompress.result.result.nanoSecPerRun);
cSize = intermediateResultCompress.result.result.sumOfReturn;
results.result.cSize = cSize;
ratio = (double)srcSize / results.result.cSize;
markNb = (markNb+1) % NB_MARKS;
DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.*f),%6.*f MB/s\r",
marks[markNb], displayName, (U32)srcSize, (U32)results.result.cSize,
ratioAccuracy, ratio,
results.result.cSpeed < (10 MB) ? 2 : 1, (double)results.result.cSpeed / (1 MB));
}
}
if(!intermediateResultDecompress.completed) {
intermediateResultDecompress = BMK_benchFunctionTimed(timeStateDecompress, &local_defaultDecompress, (void*)(dctx), &local_initDCtx, (void*)&dctxprep,
nbBlocks, (const void* const*)cPtrs, cSizes, resPtrs, resSizes, NULL);
if(intermediateResultDecompress.result.error) {
results.error = intermediateResultDecompress.result.error;
return results;
}
{
int const ratioAccuracy = (ratio < 10.) ? 3 : 2;
results.result.dSpeed = (srcSize * TIMELOOP_NANOSEC/ intermediateResultDecompress.result.result.nanoSecPerRun);
markNb = (markNb+1) % NB_MARKS;
DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.*f),%6.*f MB/s ,%6.1f MB/s \r",
marks[markNb], displayName, (U32)srcSize, (U32)results.result.cSize,
ratioAccuracy, ratio,
results.result.cSpeed < (10 MB) ? 2 : 1, (double)results.result.cSpeed / (1 MB),
(double)results.result.dSpeed / (1 MB));
}
}
while (!(compressionCompleted && decompressionCompleted)) {
if (!compressionCompleted) {
BMK_runOutcome_t const cOutcome =
BMK_benchTimedFn( timeStateCompress,
&local_defaultCompress, cctx,
&local_initCCtx, &cctxprep,
nbBlocks,
srcPtrs, srcSizes,
cPtrs, cCapacities,
cSizes);
if (!BMK_isSuccessful_runOutcome(cOutcome)) {
return BMK_benchOutcome_error();
}
} else { //iterMode;
if(adv->mode != BMK_decodeOnly) {
{ BMK_runTime_t const cResult = BMK_extract_runTime(cOutcome);
cSize = cResult.sumOfReturn;
ratio = (double)srcSize / cSize;
{ BMK_benchResult_t newResult;
newResult.cSpeed = ((U64)srcSize * TIMELOOP_NANOSEC / cResult.nanoSecPerRun);
benchResult.cSize = cSize;
if (newResult.cSpeed > benchResult.cSpeed)
benchResult.cSpeed = newResult.cSpeed;
} }
BMK_customReturn_t compressionResults = BMK_benchFunction(&local_defaultCompress, (void*)ctx, &local_initCCtx, (void*)&cctxprep,
nbBlocks, srcPtrs, srcSizes, cPtrs, cCapacities, cSizes, adv->nbSeconds);
if(compressionResults.error) {
results.error = compressionResults.error;
return results;
}
if(compressionResults.result.nanoSecPerRun == 0) {
results.result.cSpeed = 0;
} else {
results.result.cSpeed = srcSize * TIMELOOP_NANOSEC / compressionResults.result.nanoSecPerRun;
}
results.result.cSize = compressionResults.result.sumOfReturn;
{
int const ratioAccuracy = (ratio < 10.) ? 3 : 2;
cSize = compressionResults.result.sumOfReturn;
results.result.cSize = cSize;
ratio = (double)srcSize / results.result.cSize;
markNb = (markNb+1) % NB_MARKS;
DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.*f),%6.*f MB/s\r",
marks[markNb], displayName, (U32)srcSize, (U32)results.result.cSize,
ratioAccuracy, ratio,
results.result.cSpeed < (10 MB) ? 2 : 1, (double)results.result.cSpeed / (1 MB));
}
}
if(adv->mode != BMK_compressOnly) {
BMK_customReturn_t decompressionResults = BMK_benchFunction(
&local_defaultDecompress, (void*)(dctx),
&local_initDCtx, (void*)&dctxprep, nbBlocks,
(const void* const*)cPtrs, cSizes, resPtrs, resSizes, NULL,
adv->nbSeconds);
if(decompressionResults.error) {
results.error = decompressionResults.error;
return results;
}
if(decompressionResults.result.nanoSecPerRun == 0) {
results.result.dSpeed = 0;
} else {
results.result.dSpeed = srcSize * TIMELOOP_NANOSEC / decompressionResults.result.nanoSecPerRun;
}
{
int const ratioAccuracy = (ratio < 10.) ? 3 : 2;
markNb = (markNb+1) % NB_MARKS;
DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.*f),%6.*f MB/s ,%6.1f MB/s \r",
marks[markNb], displayName, (U32)srcSize, (U32)results.result.cSize,
ratioAccuracy, ratio,
results.result.cSpeed < (10 MB) ? 2 : 1, (double)results.result.cSpeed / (1 MB),
(double)results.result.dSpeed / (1 MB));
}
{ int const ratioAccuracy = (ratio < 10.) ? 3 : 2;
markNb = (markNb+1) % NB_MARKS;
DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.*f),%6.*f MB/s\r",
marks[markNb], displayName,
(U32)srcSize, (U32)cSize,
ratioAccuracy, ratio,
benchResult.cSpeed < (10 MB) ? 2 : 1, (double)benchResult.cSpeed / MB_UNIT);
}
compressionCompleted = BMK_isCompleted_TimedFn(timeStateCompress);
}
}
if(!decompressionCompleted) {
BMK_runOutcome_t const dOutcome =
BMK_benchTimedFn(timeStateDecompress,
&local_defaultDecompress, dctx,
&local_initDCtx, &dctxprep,
nbBlocks,
(const void *const *)cPtrs, cSizes,
resPtrs, resSizes,
NULL);
if(!BMK_isSuccessful_runOutcome(dOutcome)) {
return BMK_benchOutcome_error();
}
{ BMK_runTime_t const dResult = BMK_extract_runTime(dOutcome);
U64 const newDSpeed = (srcSize * TIMELOOP_NANOSEC / dResult.nanoSecPerRun);
if (newDSpeed > benchResult.dSpeed)
benchResult.dSpeed = newDSpeed;
}
{ int const ratioAccuracy = (ratio < 10.) ? 3 : 2;
markNb = (markNb+1) % NB_MARKS;
DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.*f),%6.*f MB/s ,%6.1f MB/s \r",
marks[markNb], displayName,
(U32)srcSize, (U32)benchResult.cSize,
ratioAccuracy, ratio,
benchResult.cSpeed < (10 MB) ? 2 : 1, (double)benchResult.cSpeed / MB_UNIT,
(double)benchResult.dSpeed / MB_UNIT);
}
decompressionCompleted = BMK_isCompleted_TimedFn(timeStateDecompress);
}
} /* while (!(compressionCompleted && decompressionCompleted)) */
/* CRC Checking */
{ void* resultBuffer = *resultBufferPtr;
{ const BYTE* resultBuffer = (const BYTE*)(*resultBufferPtr);
U64 const crcCheck = XXH64(resultBuffer, srcSize, 0);
/* adv->mode == 0 -> compress + decompress */
if ((adv->mode == BMK_both) && (crcOrig!=crcCheck)) {
size_t u;
DISPLAY("!!! WARNING !!! %14s : Invalid Checksum : %x != %x \n", displayName, (unsigned)crcOrig, (unsigned)crcCheck);
for (u=0; u<srcSize; u++) {
if (((const BYTE*)srcBuffer)[u] != ((const BYTE*)resultBuffer)[u]) {
if (((const BYTE*)srcBuffer)[u] != resultBuffer[u]) {
U32 segNb, bNb, pos;
size_t bacc = 0;
DISPLAY("Decoding error at pos %u ", (U32)u);
@@ -674,44 +728,47 @@ static BMK_return_t BMK_benchMemAdvancedNoAlloc(
for (n=1; n<3; n++) DISPLAY("%02X ", ((const BYTE*)srcBuffer)[u+n]);
DISPLAY(" \n");
DISPLAY("decode: ");
for (n=-5; n<0; n++) DISPLAY("%02X ", ((const BYTE*)resultBuffer)[u+n]);
DISPLAY(" :%02X: ", ((const BYTE*)resultBuffer)[u]);
for (n=1; n<3; n++) DISPLAY("%02X ", ((const BYTE*)resultBuffer)[u+n]);
for (n=-5; n<0; n++) DISPLAY("%02X ", resultBuffer[u+n]);
DISPLAY(" :%02X: ", resultBuffer[u]);
for (n=1; n<3; n++) DISPLAY("%02X ", resultBuffer[u+n]);
DISPLAY(" \n");
}
break;
}
if (u==srcSize-1) { /* should never happen */
DISPLAY("no difference detected\n");
}
}
}
}
}
} /* CRC Checking */
if (displayLevel == 1) { /* hidden display mode -q, used by python speed benchmark */
double const cSpeed = (double)results.result.cSpeed / (1 MB);
double const dSpeed = (double)results.result.dSpeed / (1 MB);
if (adv->additionalParam) {
DISPLAY("-%-3i%11i (%5.3f) %6.2f MB/s %6.1f MB/s %s (param=%d)\n", cLevel, (int)cSize, ratio, cSpeed, dSpeed, displayName, adv->additionalParam);
} else {
DISPLAY("-%-3i%11i (%5.3f) %6.2f MB/s %6.1f MB/s %s\n", cLevel, (int)cSize, ratio, cSpeed, dSpeed, displayName);
if (displayLevel == 1) { /* hidden display mode -q, used by python speed benchmark */
double const cSpeed = (double)benchResult.cSpeed / MB_UNIT;
double const dSpeed = (double)benchResult.dSpeed / MB_UNIT;
if (adv->additionalParam) {
DISPLAY("-%-3i%11i (%5.3f) %6.2f MB/s %6.1f MB/s %s (param=%d)\n", cLevel, (int)cSize, ratio, cSpeed, dSpeed, displayName, adv->additionalParam);
} else {
DISPLAY("-%-3i%11i (%5.3f) %6.2f MB/s %6.1f MB/s %s\n", cLevel, (int)cSize, ratio, cSpeed, dSpeed, displayName);
}
}
}
DISPLAYLEVEL(2, "%2i#\n", cLevel);
DISPLAYLEVEL(2, "%2i#\n", cLevel);
} /* Bench */
results.result.cMem = (1ULL << (comprParams->windowLog)) + ZSTD_sizeof_CCtx(ctx);
results.error = 0;
return results;
benchResult.cMem = (1ULL << (comprParams->windowLog)) + ZSTD_sizeof_CCtx(cctx);
return BMK_benchOutcome_setValidResult(benchResult);
}
BMK_return_t BMK_benchMemAdvanced(const void* srcBuffer, size_t srcSize,
void* dstBuffer, size_t dstCapacity,
BMK_benchOutcome_t BMK_benchMemAdvanced(const void* srcBuffer, size_t srcSize,
void* dstBuffer, size_t dstCapacity,
const size_t* fileSizes, unsigned nbFiles,
const int cLevel, const ZSTD_compressionParameters* comprParams,
int cLevel, const ZSTD_compressionParameters* comprParams,
const void* dictBuffer, size_t dictBufferSize,
int displayLevel, const char* displayName, const BMK_advancedParams_t* adv)
{
int const dstParamsError = !dstBuffer ^ !dstCapacity; /* must be both NULL or none */
size_t const blockSize = ((adv->blockSize>=32 && (adv->mode != BMK_decodeOnly)) ? adv->blockSize : srcSize) + (!srcSize) /* avoid div by 0 */ ;
U32 const maxNbBlocks = (U32) ((srcSize + (blockSize-1)) / blockSize) + nbFiles;
@@ -727,71 +784,78 @@ BMK_return_t BMK_benchMemAdvanced(const void* srcBuffer, size_t srcSize,
void ** const resPtrs = (void**)malloc(maxNbBlocks * sizeof(void*));
size_t* const resSizes = (size_t*)malloc(maxNbBlocks * sizeof(size_t));
BMK_timedFnState_t* timeStateCompress = BMK_createTimeState(adv->nbSeconds);
BMK_timedFnState_t* timeStateDecompress = BMK_createTimeState(adv->nbSeconds);
BMK_timedFnState_t* timeStateCompress = BMK_createTimedFnState(adv->nbSeconds * 1000, BMK_RUNTEST_DEFAULT_MS);
BMK_timedFnState_t* timeStateDecompress = BMK_createTimedFnState(adv->nbSeconds * 1000, BMK_RUNTEST_DEFAULT_MS);
ZSTD_CCtx* ctx = ZSTD_createCCtx();
ZSTD_DCtx* dctx = ZSTD_createDCtx();
ZSTD_CCtx* const cctx = ZSTD_createCCtx();
ZSTD_DCtx* const dctx = ZSTD_createDCtx();
const size_t maxCompressedSize = dstCapacity ? dstCapacity : ZSTD_compressBound(srcSize) + (maxNbBlocks * 1024);
void* const internalDstBuffer = dstBuffer ? NULL : malloc(maxCompressedSize);
void* const compressedBuffer = dstBuffer ? dstBuffer : internalDstBuffer;
BMK_return_t results = { { 0, 0, 0, 0 }, 0 };
int parametersConflict = !dstBuffer ^ !dstCapacity;
BMK_benchOutcome_t outcome = BMK_benchOutcome_error(); /* error by default */
void* resultBuffer = srcSize ? malloc(srcSize) : NULL;
int allocationincomplete = !srcPtrs || !srcSizes || !cPtrs ||
!cSizes || !cCapacities || !resPtrs || !resSizes ||
!timeStateCompress || !timeStateDecompress || !compressedBuffer || !resultBuffer;
int allocationincomplete = !srcPtrs || !srcSizes || !cPtrs ||
!cSizes || !cCapacities || !resPtrs || !resSizes ||
!timeStateCompress || !timeStateDecompress ||
!cctx || !dctx ||
!compressedBuffer || !resultBuffer;
if (!allocationincomplete && !parametersConflict) {
results = BMK_benchMemAdvancedNoAlloc(srcPtrs, srcSizes, cPtrs, cCapacities, cSizes,
resPtrs, resSizes, &resultBuffer, compressedBuffer, maxCompressedSize, timeStateCompress, timeStateDecompress,
srcBuffer, srcSize, fileSizes, nbFiles, cLevel, comprParams,
dictBuffer, dictBufferSize, ctx, dctx, displayLevel, displayName, adv);
if (!allocationincomplete && !dstParamsError) {
outcome = BMK_benchMemAdvancedNoAlloc(srcPtrs, srcSizes,
cPtrs, cCapacities, cSizes,
resPtrs, resSizes,
&resultBuffer,
compressedBuffer, maxCompressedSize,
timeStateCompress, timeStateDecompress,
srcBuffer, srcSize,
fileSizes, nbFiles,
cLevel, comprParams,
dictBuffer, dictBufferSize,
cctx, dctx,
displayLevel, displayName, adv);
}
/* clean up */
BMK_freeTimeState(timeStateCompress);
BMK_freeTimeState(timeStateDecompress);
ZSTD_freeCCtx(ctx);
BMK_freeTimedFnState(timeStateCompress);
BMK_freeTimedFnState(timeStateDecompress);
ZSTD_freeCCtx(cctx);
ZSTD_freeDCtx(dctx);
free(internalDstBuffer);
free(resultBuffer);
free((void*)srcPtrs);
free(srcSizes);
free(cPtrs);
free((void*)srcPtrs);
free(srcSizes);
free(cPtrs);
free(cSizes);
free(cCapacities);
free(resPtrs);
free(resSizes);
if(allocationincomplete) {
EXM_THROW(31, BMK_return_t, "allocation error : not enough memory");
RETURN_ERROR(31, BMK_benchOutcome_t, "allocation error : not enough memory");
}
if(parametersConflict) {
EXM_THROW(32, BMK_return_t, "Conflicting input results");
if(dstParamsError) {
RETURN_ERROR(32, BMK_benchOutcome_t, "Dst parameters not coherent");
}
return results;
return outcome;
}
BMK_return_t BMK_benchMem(const void* srcBuffer, size_t srcSize,
BMK_benchOutcome_t BMK_benchMem(const void* srcBuffer, size_t srcSize,
const size_t* fileSizes, unsigned nbFiles,
const int cLevel, const ZSTD_compressionParameters* comprParams,
int cLevel, const ZSTD_compressionParameters* comprParams,
const void* dictBuffer, size_t dictBufferSize,
int displayLevel, const char* displayName) {
const BMK_advancedParams_t adv = BMK_initAdvancedParams();
BMK_advancedParams_t const adv = BMK_initAdvancedParams();
return BMK_benchMemAdvanced(srcBuffer, srcSize,
NULL, 0,
fileSizes, nbFiles,
@@ -800,6 +864,71 @@ BMK_return_t BMK_benchMem(const void* srcBuffer, size_t srcSize,
displayLevel, displayName, &adv);
}
static BMK_benchOutcome_t BMK_benchCLevel(const void* srcBuffer, size_t benchedSize,
const size_t* fileSizes, unsigned nbFiles,
int cLevel, const ZSTD_compressionParameters* comprParams,
const void* dictBuffer, size_t dictBufferSize,
int displayLevel, const char* displayName,
BMK_advancedParams_t const * const adv)
{
const char* pch = strrchr(displayName, '\\'); /* Windows */
if (!pch) pch = strrchr(displayName, '/'); /* Linux */
if (pch) displayName = pch+1;
if (adv->realTime) {
DISPLAYLEVEL(2, "Note : switching to real-time priority \n");
SET_REALTIME_PRIORITY;
}
if (displayLevel == 1 && !adv->additionalParam) /* --quiet mode */
DISPLAY("bench %s %s: input %u bytes, %u seconds, %u KB blocks\n",
ZSTD_VERSION_STRING, ZSTD_GIT_COMMIT_STRING,
(U32)benchedSize, adv->nbSeconds, (U32)(adv->blockSize>>10));
return BMK_benchMemAdvanced(srcBuffer, benchedSize,
NULL, 0,
fileSizes, nbFiles,
cLevel, comprParams,
dictBuffer, dictBufferSize,
displayLevel, displayName, adv);
}
BMK_benchOutcome_t BMK_syntheticTest(int cLevel, double compressibility,
const ZSTD_compressionParameters* compressionParams,
int displayLevel, const BMK_advancedParams_t* adv)
{
char name[20] = {0};
size_t const benchedSize = 10000000;
void* srcBuffer;
BMK_benchOutcome_t res;
if (cLevel > ZSTD_maxCLevel()) {
RETURN_ERROR(15, BMK_benchOutcome_t, "Invalid Compression Level");
}
/* Memory allocation */
srcBuffer = malloc(benchedSize);
if (!srcBuffer) RETURN_ERROR(21, BMK_benchOutcome_t, "not enough memory");
/* Fill input buffer */
RDG_genBuffer(srcBuffer, benchedSize, compressibility, 0.0, 0);
/* Bench */
snprintf (name, sizeof(name), "Synthetic %2u%%", (unsigned)(compressibility*100));
res = BMK_benchCLevel(srcBuffer, benchedSize,
&benchedSize /* ? */, 1 /* ? */,
cLevel, compressionParams,
NULL, 0, /* dictionary */
displayLevel, name, adv);
/* clean up */
free(srcBuffer);
return res;
}
static size_t BMK_findMaxMem(U64 requiredMem)
{
size_t const step = 64 MB;
@@ -818,45 +947,13 @@ static size_t BMK_findMaxMem(U64 requiredMem)
return (size_t)(requiredMem);
}
static BMK_return_t BMK_benchCLevel(const void* srcBuffer, size_t benchedSize,
const size_t* fileSizes, unsigned nbFiles,
const int cLevel, const ZSTD_compressionParameters* comprParams,
const void* dictBuffer, size_t dictBufferSize,
int displayLevel, const char* displayName,
BMK_advancedParams_t const * const adv)
{
BMK_return_t res;
const char* pch = strrchr(displayName, '\\'); /* Windows */
if (!pch) pch = strrchr(displayName, '/'); /* Linux */
if (pch) displayName = pch+1;
if (adv->realTime) {
DISPLAYLEVEL(2, "Note : switching to real-time priority \n");
SET_REALTIME_PRIORITY;
}
if (displayLevel == 1 && !adv->additionalParam)
DISPLAY("bench %s %s: input %u bytes, %u seconds, %u KB blocks\n", ZSTD_VERSION_STRING, ZSTD_GIT_COMMIT_STRING, (U32)benchedSize, adv->nbSeconds, (U32)(adv->blockSize>>10));
res = BMK_benchMemAdvanced(srcBuffer, benchedSize,
NULL, 0,
fileSizes, nbFiles,
cLevel, comprParams,
dictBuffer, dictBufferSize,
displayLevel, displayName, adv);
return res;
}
/*! BMK_loadFiles() :
* Loads `buffer` with content of files listed within `fileNamesTable`.
* At most, fills `buffer` entirely. */
static int BMK_loadFiles(void* buffer, size_t bufferSize,
size_t* fileSizes, const char* const * const fileNamesTable,
unsigned nbFiles, int displayLevel)
size_t* fileSizes,
const char* const * fileNamesTable, unsigned nbFiles,
int displayLevel)
{
size_t pos = 0, totalSize = 0;
unsigned n;
@@ -877,9 +974,10 @@ static int BMK_loadFiles(void* buffer, size_t bufferSize,
if (f==NULL) EXM_THROW_INT(10, "impossible to open file %s", fileNamesTable[n]);
DISPLAYUPDATE(2, "Loading %s... \r", fileNamesTable[n]);
if (fileSize > bufferSize-pos) fileSize = bufferSize-pos, nbFiles=n; /* buffer too small - stop after this file */
{ size_t const readSize = fread(((char*)buffer)+pos, 1, (size_t)fileSize, f);
if (readSize != (size_t)fileSize) EXM_THROW_INT(11, "could not read %s", fileNamesTable[n]);
pos += readSize; }
{ size_t const readSize = fread(((char*)buffer)+pos, 1, (size_t)fileSize, f);
if (readSize != (size_t)fileSize) EXM_THROW_INT(11, "could not read %s", fileNamesTable[n]);
pos += readSize;
}
fileSizes[n] = (size_t)fileSize;
totalSize += (size_t)fileSize;
fclose(f);
@@ -889,50 +987,53 @@ static int BMK_loadFiles(void* buffer, size_t bufferSize,
return 0;
}
BMK_return_t BMK_benchFilesAdvanced(const char* const * const fileNamesTable, unsigned const nbFiles,
const char* const dictFileName, int const cLevel,
const ZSTD_compressionParameters* const compressionParams,
int displayLevel, const BMK_advancedParams_t * const adv)
BMK_benchOutcome_t BMK_benchFilesAdvanced(
const char* const * fileNamesTable, unsigned nbFiles,
const char* dictFileName, int cLevel,
const ZSTD_compressionParameters* compressionParams,
int displayLevel, const BMK_advancedParams_t* adv)
{
void* srcBuffer = NULL;
size_t benchedSize;
void* dictBuffer = NULL;
size_t dictBufferSize = 0;
size_t* fileSizes = NULL;
BMK_return_t res;
BMK_benchOutcome_t res;
U64 const totalSizeToLoad = UTIL_getTotalFileSize(fileNamesTable, nbFiles);
if(!nbFiles) {
EXM_THROW(14, BMK_return_t, "No Files to Benchmark");
if (!nbFiles) {
RETURN_ERROR(14, BMK_benchOutcome_t, "No Files to Benchmark");
}
if (cLevel > ZSTD_maxCLevel()) {
EXM_THROW(15, BMK_return_t, "Invalid Compression Level");
RETURN_ERROR(15, BMK_benchOutcome_t, "Invalid Compression Level");
}
fileSizes = (size_t*)calloc(nbFiles, sizeof(size_t));
if (!fileSizes) EXM_THROW(12, BMK_return_t, "not enough memory for fileSizes");
if (!fileSizes) RETURN_ERROR(12, BMK_benchOutcome_t, "not enough memory for fileSizes");
/* Load dictionary */
if (dictFileName != NULL) {
U64 const dictFileSize = UTIL_getFileSize(dictFileName);
if (dictFileSize > 64 MB) {
free(fileSizes);
EXM_THROW(10, BMK_return_t, "dictionary file %s too large", dictFileName);
RETURN_ERROR(10, BMK_benchOutcome_t, "dictionary file %s too large", dictFileName);
}
dictBufferSize = (size_t)dictFileSize;
dictBuffer = malloc(dictBufferSize);
if (dictBuffer==NULL) {
free(fileSizes);
EXM_THROW(11, BMK_return_t, "not enough memory for dictionary (%u bytes)",
RETURN_ERROR(11, BMK_benchOutcome_t, "not enough memory for dictionary (%u bytes)",
(U32)dictBufferSize);
}
{
int errorCode = BMK_loadFiles(dictBuffer, dictBufferSize, fileSizes, &dictFileName, 1, displayLevel);
if(errorCode) {
res.error = errorCode;
{ int const errorCode = BMK_loadFiles(dictBuffer, dictBufferSize,
fileSizes, &dictFileName /*?*/,
1 /*?*/, displayLevel);
if (errorCode) {
res = BMK_benchOutcome_error();
goto _cleanUp;
}
}
} }
}
/* Memory allocation & restrictions */
@@ -945,29 +1046,28 @@ BMK_return_t BMK_benchFilesAdvanced(const char* const * const fileNamesTable, un
if (!srcBuffer) {
free(dictBuffer);
free(fileSizes);
EXM_THROW(12, BMK_return_t, "not enough memory");
RETURN_ERROR(12, BMK_benchOutcome_t, "not enough memory");
}
/* Load input buffer */
{
int errorCode = BMK_loadFiles(srcBuffer, benchedSize, fileSizes, fileNamesTable, nbFiles, displayLevel);
if(errorCode) {
res.error = errorCode;
{ int const errorCode = BMK_loadFiles(srcBuffer, benchedSize,
fileSizes, fileNamesTable, nbFiles,
displayLevel);
if (errorCode) {
res = BMK_benchOutcome_error();
goto _cleanUp;
}
}
} }
/* Bench */
{
char mfName[20] = {0};
{ char mfName[20] = {0};
snprintf (mfName, sizeof(mfName), " %u files", nbFiles);
{
const char* const displayName = (nbFiles > 1) ? mfName : fileNamesTable[0];
res = BMK_benchCLevel(srcBuffer, benchedSize,
fileSizes, nbFiles,
cLevel, compressionParams,
dictBuffer, dictBufferSize,
displayLevel, displayName,
adv);
{ const char* const displayName = (nbFiles > 1) ? mfName : fileNamesTable[0];
res = BMK_benchCLevel(srcBuffer, benchedSize,
fileSizes, nbFiles,
cLevel, compressionParams,
dictBuffer, dictBufferSize,
displayLevel, displayName,
adv);
} }
_cleanUp:
@@ -978,44 +1078,12 @@ _cleanUp:
}
BMK_return_t BMK_syntheticTest(int cLevel, double compressibility,
const ZSTD_compressionParameters* compressionParams,
int displayLevel, const BMK_advancedParams_t * const adv)
BMK_benchOutcome_t BMK_benchFiles(
const char* const * fileNamesTable, unsigned nbFiles,
const char* dictFileName,
int cLevel, const ZSTD_compressionParameters* compressionParams,
int displayLevel)
{
char name[20] = {0};
size_t benchedSize = 10000000;
void* srcBuffer;
BMK_return_t res;
if (cLevel > ZSTD_maxCLevel()) {
EXM_THROW(15, BMK_return_t, "Invalid Compression Level");
}
/* Memory allocation */
srcBuffer = malloc(benchedSize);
if (!srcBuffer) EXM_THROW(21, BMK_return_t, "not enough memory");
/* Fill input buffer */
RDG_genBuffer(srcBuffer, benchedSize, compressibility, 0.0, 0);
/* Bench */
snprintf (name, sizeof(name), "Synthetic %2u%%", (unsigned)(compressibility*100));
res = BMK_benchCLevel(srcBuffer, benchedSize,
&benchedSize, 1,
cLevel, compressionParams,
NULL, 0,
displayLevel, name, adv);
/* clean up */
free(srcBuffer);
return res;
}
BMK_return_t BMK_benchFiles(const char* const * const fileNamesTable, unsigned const nbFiles,
const char* const dictFileName,
int const cLevel, const ZSTD_compressionParameters* const compressionParams,
int displayLevel) {
const BMK_advancedParams_t adv = BMK_initAdvancedParams();
BMK_advancedParams_t const adv = BMK_initAdvancedParams();
return BMK_benchFilesAdvanced(fileNamesTable, nbFiles, dictFileName, cLevel, compressionParams, displayLevel, &adv);
}
programs/bench.h
+208 -132
View File
@@ -15,59 +15,82 @@ extern "C" {
#ifndef BENCH_H_121279284357
#define BENCH_H_121279284357
/* === Dependencies === */
#include <stddef.h> /* size_t */
#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_compressionParameters */
#include "zstd.h" /* ZSTD_compressionParameters */
/* Creates a struct of type typeName with an int type .error field
* and a .result field of some baseType. Functions with return
* typeName pass a successful result with .error = 0 and .result
* with the intended result, while returning an error will result
* in .error != 0.
/* === Constants === */
#define MB_UNIT 1000000
/* === Benchmark functions === */
/* Creates a variant `typeName`, able to express "error or valid result".
* Functions with return type `typeName`
* must first check if result is valid, using BMK_isSuccessful_*(),
* and only then can extract `baseType`.
*/
#define ERROR_STRUCT(baseType, typeName) typedef struct { \
baseType result; \
int error; \
} typeName
#define VARIANT_ERROR_RESULT(baseType, variantName) \
\
typedef struct { \
baseType internal_never_use_directly; \
int tag; \
} variantName
typedef struct {
size_t cSize;
U64 cSpeed; /* bytes / sec */
U64 dSpeed;
size_t cMem;
} BMK_result_t;
unsigned long long cSpeed; /* bytes / sec */
unsigned long long dSpeed;
size_t cMem; /* ? what is reported ? */
} BMK_benchResult_t;
ERROR_STRUCT(BMK_result_t, BMK_return_t);
VARIANT_ERROR_RESULT(BMK_benchResult_t, BMK_benchOutcome_t);
/* called in cli */
/* Loads files in fileNamesTable into memory, as well as a dictionary
* from dictFileName, and then uses benchMem */
/* fileNamesTable - name of files to benchmark
* nbFiles - number of files (size of fileNamesTable), must be > 0
* dictFileName - name of dictionary file to load
* cLevel - compression level to benchmark, errors if invalid
* compressionParams - basic compression Parameters
* displayLevel - what gets printed
* 0 : no display;
* 1 : errors;
* 2 : + result + interaction + warnings;
* 3 : + progression;
* 4 : + information
* return
* .error will give a nonzero error value if an error has occured
* .result - if .error = 0, .result will return the time taken to compression speed
* (.cSpeed), decompression speed (.dSpeed), and compressed size (.cSize) of the original
* file
/* check first if the return structure represents an error or a valid result */
int BMK_isSuccessful_benchOutcome(BMK_benchOutcome_t outcome);
/* extract result from variant type.
* note : this function will abort() program execution if result is not valid
* check result validity first, by using BMK_isSuccessful_benchOutcome()
*/
BMK_return_t BMK_benchFiles(const char* const * const fileNamesTable, unsigned const nbFiles,
const char* const dictFileName,
int const cLevel, const ZSTD_compressionParameters* const compressionParams,
BMK_benchResult_t BMK_extract_benchResult(BMK_benchOutcome_t outcome);
/*! BMK_benchFiles() -- called by zstdcli */
/* Loads files from fileNamesTable into memory,
* and an optional dictionary from dictFileName (can be NULL),
* then uses benchMem().
* fileNamesTable - name of files to benchmark.
* nbFiles - number of files (size of fileNamesTable), must be > 0.
* dictFileName - name of dictionary file to load.
* cLevel - compression level to benchmark, errors if invalid.
* compressionParams - advanced compression Parameters.
* displayLevel - what gets printed:
* 0 : no display;
* 1 : errors;
* 2 : + result + interaction + warnings;
* 3 : + information;
* 4 : + debug
* @return:
* a variant, which expresses either an error, or a valid result.
* Use BMK_isSuccessful_benchOutcome() to check if function was successful.
* If yes, extract the valid result with BMK_extract_benchResult(),
* it will contain :
* .cSpeed: compression speed in bytes per second,
* .dSpeed: decompression speed in bytes per second,
* .cSize : compressed size, in bytes
* .cMem : memory budget required for the compression context
*/
BMK_benchOutcome_t BMK_benchFiles(
const char* const * fileNamesTable, unsigned nbFiles,
const char* dictFileName,
int cLevel, const ZSTD_compressionParameters* compressionParams,
int displayLevel);
typedef enum {
BMK_timeMode = 0,
BMK_iterMode = 1
} BMK_loopMode_t;
typedef enum {
BMK_both = 0,
@@ -77,15 +100,14 @@ typedef enum {
typedef struct {
BMK_mode_t mode; /* 0: all, 1: compress only 2: decode only */
BMK_loopMode_t loopMode; /* if loopmode, then nbSeconds = nbLoops */
unsigned nbSeconds; /* default timing is in nbSeconds */
size_t blockSize; /* Maximum allowable size of a block*/
size_t blockSize; /* Maximum size of each block*/
unsigned nbWorkers; /* multithreading */
unsigned realTime; /* real time priority */
int additionalParam; /* used by python speed benchmark */
unsigned ldmFlag; /* enables long distance matching */
unsigned ldmMinMatch; /* below: parameters for long distance matching, see zstd.1.md for meaning */
unsigned ldmHashLog;
unsigned ldmMinMatch; /* below: parameters for long distance matching, see zstd.1.md */
unsigned ldmHashLog;
unsigned ldmBucketSizeLog;
unsigned ldmHashEveryLog;
} BMK_advancedParams_t;
@@ -93,132 +115,186 @@ typedef struct {
/* returns default parameters used by nonAdvanced functions */
BMK_advancedParams_t BMK_initAdvancedParams(void);
/* See benchFiles for normal parameter uses and return, see advancedParams_t for adv */
BMK_return_t BMK_benchFilesAdvanced(const char* const * const fileNamesTable, unsigned const nbFiles,
const char* const dictFileName,
int const cLevel, const ZSTD_compressionParameters* const compressionParams,
int displayLevel, const BMK_advancedParams_t* const adv);
/*! BMK_benchFilesAdvanced():
* Same as BMK_benchFiles(),
* with more controls, provided through advancedParams_t structure */
BMK_benchOutcome_t BMK_benchFilesAdvanced(
const char* const * fileNamesTable, unsigned nbFiles,
const char* dictFileName,
int cLevel, const ZSTD_compressionParameters* compressionParams,
int displayLevel, const BMK_advancedParams_t* adv);
/* called in cli */
/* Generates a sample with datagen with the compressibility argument*/
/* cLevel - compression level to benchmark, errors if invalid
* compressibility - determines compressibility of sample
* compressionParams - basic compression Parameters
* displayLevel - see benchFiles
* adv - see advanced_Params_t
* return
* .error will give a nonzero error value if an error has occured
* .result - if .error = 0, .result will return the time taken to compression speed
* (.cSpeed), decompression speed (.dSpeed), and compressed size (.cSize) of the original
* file
/*! BMK_syntheticTest() -- called from zstdcli */
/* Generates a sample with datagen, using compressibility argument */
/* cLevel - compression level to benchmark, errors if invalid
* compressibility - determines compressibility of sample
* compressionParams - basic compression Parameters
* displayLevel - see benchFiles
* adv - see advanced_Params_t
* @return:
* a variant, which expresses either an error, or a valid result.
* Use BMK_isSuccessful_benchOutcome() to check if function was successful.
* If yes, extract the valid result with BMK_extract_benchResult(),
* it will contain :
* .cSpeed: compression speed in bytes per second,
* .dSpeed: decompression speed in bytes per second,
* .cSize : compressed size, in bytes
* .cMem : memory budget required for the compression context
*/
BMK_return_t BMK_syntheticTest(int cLevel, double compressibility,
BMK_benchOutcome_t BMK_syntheticTest(
int cLevel, double compressibility,
const ZSTD_compressionParameters* compressionParams,
int displayLevel, const BMK_advancedParams_t * const adv);
int displayLevel, const BMK_advancedParams_t* adv);
/* basic benchmarking function, called in paramgrill
* applies ZSTD_compress_generic() and ZSTD_decompress_generic() on data in srcBuffer
* with specific compression parameters specified by other arguments using benchFunction
* (cLevel, comprParams + adv in advanced Mode) */
/* srcBuffer - data source, expected to be valid compressed data if in Decode Only Mode
* srcSize - size of data in srcBuffer
* cLevel - compression level
* comprParams - basic compression parameters
* dictBuffer - a dictionary if used, null otherwise
* dictBufferSize - size of dictBuffer, 0 otherwise
* diplayLevel - see BMK_benchFiles
* displayName - name used by display
* return
* .error will give a nonzero value if an error has occured
* .result - if .error = 0, will give the same results as benchFiles
* but for the data stored in srcBuffer
/* === Benchmark Zstandard in a memory-to-memory scenario === */
/** BMK_benchMem() -- core benchmarking function, called in paramgrill
* applies ZSTD_compress_generic() and ZSTD_decompress_generic() on data in srcBuffer
* with specific compression parameters provided by other arguments using benchFunction
* (cLevel, comprParams + adv in advanced Mode) */
/* srcBuffer - data source, expected to be valid compressed data if in Decode Only Mode
* srcSize - size of data in srcBuffer
* fileSizes - srcBuffer is considered cut into 1+ segments, to compress separately.
* note : sum(fileSizes) must be == srcSize. (<== ensure it's properly checked)
* nbFiles - nb of segments
* cLevel - compression level
* comprParams - basic compression parameters
* dictBuffer - a dictionary if used, null otherwise
* dictBufferSize - size of dictBuffer, 0 otherwise
* diplayLevel - see BMK_benchFiles
* displayName - name used by display
* @return:
* a variant, which expresses either an error, or a valid result.
* Use BMK_isSuccessful_benchOutcome() to check if function was successful.
* If yes, extract the valid result with BMK_extract_benchResult(),
* it will contain :
* .cSpeed: compression speed in bytes per second,
* .dSpeed: decompression speed in bytes per second,
* .cSize : compressed size, in bytes
* .cMem : memory budget required for the compression context
*/
BMK_return_t BMK_benchMem(const void* srcBuffer, size_t srcSize,
BMK_benchOutcome_t BMK_benchMem(const void* srcBuffer, size_t srcSize,
const size_t* fileSizes, unsigned nbFiles,
const int cLevel, const ZSTD_compressionParameters* comprParams,
int cLevel, const ZSTD_compressionParameters* comprParams,
const void* dictBuffer, size_t dictBufferSize,
int displayLevel, const char* displayName);
/* See benchMem for normal parameter uses and return, see advancedParams_t for adv
/* BMK_benchMemAdvanced() : same as BMK_benchMem()
* with following additional options :
* dstBuffer - destination buffer to write compressed output in, NULL if none provided.
* dstCapacity - capacity of destination buffer, give 0 if dstBuffer = NULL
* adv = see advancedParams_t
*/
BMK_return_t BMK_benchMemAdvanced(const void* srcBuffer, size_t srcSize,
void* dstBuffer, size_t dstCapacity,
BMK_benchOutcome_t BMK_benchMemAdvanced(const void* srcBuffer, size_t srcSize,
void* dstBuffer, size_t dstCapacity,
const size_t* fileSizes, unsigned nbFiles,
const int cLevel, const ZSTD_compressionParameters* comprParams,
int cLevel, const ZSTD_compressionParameters* comprParams,
const void* dictBuffer, size_t dictBufferSize,
int displayLevel, const char* displayName,
const BMK_advancedParams_t* adv);
/* ==== Benchmarking any function, iterated on a set of blocks ==== */
typedef struct {
size_t sumOfReturn; /* sum of return values */
U64 nanoSecPerRun; /* time per iteration */
} BMK_customResult_t;
unsigned long long nanoSecPerRun; /* time per iteration */
size_t sumOfReturn; /* sum of return values */
} BMK_runTime_t;
ERROR_STRUCT(BMK_customResult_t, BMK_customReturn_t);
VARIANT_ERROR_RESULT(BMK_runTime_t, BMK_runOutcome_t);
typedef size_t (*BMK_benchFn_t)(const void*, size_t, void*, size_t, void*);
typedef size_t (*BMK_initFn_t)(void*);
/* check first if the return structure represents an error or a valid result */
int BMK_isSuccessful_runOutcome(BMK_runOutcome_t outcome);
/* This function times the execution of 2 argument functions, benchFn and initFn */
/* extract result from variant type.
* note : this function will abort() program execution if result is not valid
* check result validity first, by using BMK_isSuccessful_runOutcome()
*/
BMK_runTime_t BMK_extract_runTime(BMK_runOutcome_t outcome);
typedef size_t (*BMK_benchFn_t)(const void* src, size_t srcSize, void* dst, size_t dstCapacity, void* customPayload);
typedef size_t (*BMK_initFn_t)(void* initPayload);
/* BMK_benchFunction() :
* This function times the execution of 2 argument functions, benchFn and initFn */
/* benchFn - (*benchFn)(srcBuffers[i], srcSizes[i], dstBuffers[i], dstCapacities[i], benchPayload)
* is run nbLoops times
* initFn - (*initFn)(initPayload) is run once per benchmark at the beginning. This argument can
* be NULL, in which case nothing is run.
* blockCount - number of blocks (size of srcBuffers, srcSizes, dstBuffers, dstCapacities)
* initFn - (*initFn)(initPayload) is run once per benchmark, at the beginning.
* This argument can be NULL, in which case nothing is run.
* blockCount - number of blocks. Size of all array parameters : srcBuffers, srcSizes, dstBuffers, dstCapacities, blockResults
* srcBuffers - an array of buffers to be operated on by benchFn
* srcSizes - an array of the sizes of above buffers
* dstBuffers - an array of buffers to be written into by benchFn
* dstCapacities - an array of the capacities of above buffers
* blockResults - the return value of benchFn called on each block.
* blockResults - Optional: store the return value of benchFn for each block. Use NULL if this result is not requested.
* nbLoops - defines number of times benchFn is run.
* assumed array of size blockCount, will have compressed size of each block written to it.
* return
* .error will give a nonzero value if ZSTD_isError() is nonzero for any of the return
* of the calls to initFn and benchFn, or if benchFunction errors internally
* .result - if .error = 0, then .result will contain the sum of all return values of
* benchFn on the first iteration through all of the blocks (.sumOfReturn) and also
* the time per run of benchFn (.nanoSecPerRun). For the former, this
* is generally intended to be used on functions which return the # of bytes written
* into dstBuffer, hence this value will be the total amount of bytes written to
* dstBuffer.
* @return: a variant, which express either an error, or can generate a valid BMK_runTime_t result.
* Use BMK_isSuccessful_runOutcome() to check if function was successful.
* If yes, extract the result with BMK_extract_runTime(),
* it will contain :
* .sumOfReturn : the sum of all return values of benchFn through all of blocks
* .nanoSecPerRun : time per run of benchFn + (time for initFn / nbLoops)
* .sumOfReturn is generally intended for functions which return a # of bytes written into dstBuffer,
* in which case, this value will be the total amount of bytes written into dstBuffer.
*/
BMK_customReturn_t BMK_benchFunction(BMK_benchFn_t benchFn, void* benchPayload,
BMK_runOutcome_t BMK_benchFunction(
BMK_benchFn_t benchFn, void* benchPayload,
BMK_initFn_t initFn, void* initPayload,
size_t blockCount,
const void* const * const srcBuffers, const size_t* srcSizes,
void * const * const dstBuffers, const size_t* dstCapacities, size_t* blockResults,
const void *const * srcBuffers, const size_t* srcSizes,
void *const * dstBuffers, const size_t* dstCapacities,
size_t* blockResults,
unsigned nbLoops);
/* state information needed to advance computation for benchFunctionTimed */
typedef struct BMK_timeState_t BMK_timedFnState_t;
/* initializes timeState object with desired number of seconds */
BMK_timedFnState_t* BMK_createTimeState(unsigned nbSeconds);
/* resets existing timeState object */
void BMK_resetTimeState(BMK_timedFnState_t*, unsigned nbSeconds);
/* deletes timeState object */
void BMK_freeTimeState(BMK_timedFnState_t* state);
typedef struct {
BMK_customReturn_t result;
int completed;
} BMK_customTimedReturn_t;
/* ==== Benchmark any function, providing intermediate results ==== */
/*
* Benchmarks custom functions like BMK_benchFunction(), but runs for nbSeconds seconds rather than a fixed number of loops
* arguments mostly the same other than BMK_benchFunction()
* Usage - benchFunctionTimed will return in approximately one second. Keep calling BMK_benchFunctionTimed() until the return's completed field = 1.
* to continue updating intermediate result. Intermediate return values are returned by the function.
/* state information tracking benchmark session */
typedef struct BMK_timedFnState_s BMK_timedFnState_t;
/* BMK_createTimedFnState() and BMK_resetTimedFnState() :
* Create/Set BMK_timedFnState_t for next benchmark session,
* which shall last a minimum of total_ms milliseconds,
* producing intermediate results, paced at interval of (approximately) run_ms.
*/
BMK_customTimedReturn_t BMK_benchFunctionTimed(BMK_timedFnState_t* cont,
BMK_benchFn_t benchFn, void* benchPayload,
BMK_initFn_t initFn, void* initPayload,
size_t blockCount,
const void* const * const srcBlockBuffers, const size_t* srcBlockSizes,
void* const * const dstBlockBuffers, const size_t* dstBlockCapacities, size_t* blockResults);
BMK_timedFnState_t* BMK_createTimedFnState(unsigned total_ms, unsigned run_ms);
void BMK_resetTimedFnState(BMK_timedFnState_t* timedFnState, unsigned total_ms, unsigned run_ms);
void BMK_freeTimedFnState(BMK_timedFnState_t* state);
/* Tells if duration of all benchmark runs has exceeded total_ms
*/
int BMK_isCompleted_TimedFn(const BMK_timedFnState_t* timedFnState);
/* BMK_benchTimedFn() :
* Similar to BMK_benchFunction(), most arguments being identical.
* Automatically determines `nbLoops` so that each result is regularly produced at interval of about run_ms.
* Note : minimum `nbLoops` is 1, therefore a run may last more than run_ms, and possibly even more than total_ms.
* Usage - initialize timedFnState, select benchmark duration (total_ms) and each measurement duration (run_ms)
* call BMK_benchTimedFn() repetitively, each measurement is supposed to last about run_ms
* Check if total time budget is spent or exceeded, using BMK_isCompleted_TimedFn()
*/
BMK_runOutcome_t BMK_benchTimedFn(
BMK_timedFnState_t* timedFnState,
BMK_benchFn_t benchFn, void* benchPayload,
BMK_initFn_t initFn, void* initPayload,
size_t blockCount,
const void *const * srcBlockBuffers, const size_t* srcBlockSizes,
void *const * dstBlockBuffers, const size_t* dstBlockCapacities,
size_t* blockResults);
#endif /* BENCH_H_121279284357 */
programs/dibio.c
+39 -13
View File
@@ -27,6 +27,7 @@
#include <string.h> /* memset */
#include <stdio.h> /* fprintf, fopen, ftello64 */
#include <errno.h> /* errno */
#include <assert.h>
#include "mem.h" /* read */
#include "error_private.h"
@@ -43,6 +44,7 @@
#define SAMPLESIZE_MAX (128 KB)
#define MEMMULT 11 /* rough estimation : memory cost to analyze 1 byte of sample */
#define COVER_MEMMULT 9 /* rough estimation : memory cost to analyze 1 byte of sample */
#define FASTCOVER_MEMMULT 1 /* rough estimation : memory cost to analyze 1 byte of sample */
static const size_t g_maxMemory = (sizeof(size_t) == 4) ? (2 GB - 64 MB) : ((size_t)(512 MB) << sizeof(size_t));
#define NOISELENGTH 32
@@ -165,6 +167,7 @@ static U32 DiB_rand(U32* src)
static void DiB_shuffle(const char** fileNamesTable, unsigned nbFiles) {
U32 seed = 0xFD2FB528;
unsigned i;
assert(nbFiles >= 1);
for (i = nbFiles - 1; i > 0; --i) {
unsigned const j = DiB_rand(&seed) % (i + 1);
const char* const tmp = fileNamesTable[j];
@@ -269,16 +272,19 @@ size_t ZDICT_trainFromBuffer_unsafe_legacy(void* dictBuffer, size_t dictBufferCa
int DiB_trainFromFiles(const char* dictFileName, unsigned maxDictSize,
const char** fileNamesTable, unsigned nbFiles, size_t chunkSize,
ZDICT_legacy_params_t *params, ZDICT_cover_params_t *coverParams,
int optimizeCover)
ZDICT_legacy_params_t* params, ZDICT_cover_params_t* coverParams,
ZDICT_fastCover_params_t* fastCoverParams, int optimize)
{
unsigned const displayLevel = params ? params->zParams.notificationLevel :
coverParams ? coverParams->zParams.notificationLevel :
fastCoverParams ? fastCoverParams->zParams.notificationLevel :
0; /* should never happen */
void* const dictBuffer = malloc(maxDictSize);
fileStats const fs = DiB_fileStats(fileNamesTable, nbFiles, chunkSize, displayLevel);
size_t* const sampleSizes = (size_t*)malloc(fs.nbSamples * sizeof(size_t));
size_t const memMult = params ? MEMMULT : COVER_MEMMULT;
size_t const memMult = params ? MEMMULT :
coverParams ? COVER_MEMMULT:
FASTCOVER_MEMMULT;
size_t const maxMem = DiB_findMaxMem(fs.totalSizeToLoad * memMult) / memMult;
size_t loadedSize = (size_t) MIN ((unsigned long long)maxMem, fs.totalSizeToLoad);
void* const srcBuffer = malloc(loadedSize+NOISELENGTH);
@@ -310,7 +316,8 @@ int DiB_trainFromFiles(const char* dictFileName, unsigned maxDictSize,
/* Load input buffer */
DISPLAYLEVEL(3, "Shuffling input files\n");
DiB_shuffle(fileNamesTable, nbFiles);
nbFiles = DiB_loadFiles(srcBuffer, &loadedSize, sampleSizes, fs.nbSamples, fileNamesTable, nbFiles, chunkSize, displayLevel);
DiB_loadFiles(srcBuffer, &loadedSize, sampleSizes, fs.nbSamples, fileNamesTable, nbFiles, chunkSize, displayLevel);
{ size_t dictSize;
if (params) {
@@ -318,17 +325,36 @@ int DiB_trainFromFiles(const char* dictFileName, unsigned maxDictSize,
dictSize = ZDICT_trainFromBuffer_unsafe_legacy(dictBuffer, maxDictSize,
srcBuffer, sampleSizes, fs.nbSamples,
*params);
} else if (optimizeCover) {
dictSize = ZDICT_optimizeTrainFromBuffer_cover(dictBuffer, maxDictSize,
srcBuffer, sampleSizes, fs.nbSamples,
coverParams);
if (!ZDICT_isError(dictSize)) {
unsigned splitPercentage = (unsigned)(coverParams->splitPoint * 100);
DISPLAYLEVEL(2, "k=%u\nd=%u\nsteps=%u\nsplit=%u\n", coverParams->k, coverParams->d, coverParams->steps, splitPercentage);
} else if (coverParams) {
if (optimize) {
dictSize = ZDICT_optimizeTrainFromBuffer_cover(dictBuffer, maxDictSize,
srcBuffer, sampleSizes, fs.nbSamples,
coverParams);
if (!ZDICT_isError(dictSize)) {
unsigned splitPercentage = (unsigned)(coverParams->splitPoint * 100);
DISPLAYLEVEL(2, "k=%u\nd=%u\nsteps=%u\nsplit=%u\n", coverParams->k, coverParams->d,
coverParams->steps, splitPercentage);
}
} else {
dictSize = ZDICT_trainFromBuffer_cover(dictBuffer, maxDictSize, srcBuffer,
sampleSizes, fs.nbSamples, *coverParams);
}
} else {
dictSize = ZDICT_trainFromBuffer_cover(dictBuffer, maxDictSize, srcBuffer,
sampleSizes, fs.nbSamples, *coverParams);
assert(fastCoverParams != NULL);
if (optimize) {
dictSize = ZDICT_optimizeTrainFromBuffer_fastCover(dictBuffer, maxDictSize,
srcBuffer, sampleSizes, fs.nbSamples,
fastCoverParams);
if (!ZDICT_isError(dictSize)) {
unsigned splitPercentage = (unsigned)(fastCoverParams->splitPoint * 100);
DISPLAYLEVEL(2, "k=%u\nd=%u\nf=%u\nsteps=%u\nsplit=%u\naccel=%u\n", fastCoverParams->k,
fastCoverParams->d, fastCoverParams->f, fastCoverParams->steps, splitPercentage,
fastCoverParams->accel);
}
} else {
dictSize = ZDICT_trainFromBuffer_fastCover(dictBuffer, maxDictSize, srcBuffer,
sampleSizes, fs.nbSamples, *fastCoverParams);
}
}
if (ZDICT_isError(dictSize)) {
DISPLAYLEVEL(1, "dictionary training failed : %s \n", ZDICT_getErrorName(dictSize)); /* should not happen */
programs/dibio.h
+2 -2
View File
@@ -33,7 +33,7 @@
*/
int DiB_trainFromFiles(const char* dictFileName, unsigned maxDictSize,
const char** fileNamesTable, unsigned nbFiles, size_t chunkSize,
ZDICT_legacy_params_t *params, ZDICT_cover_params_t *coverParams,
int optimizeCover);
ZDICT_legacy_params_t* params, ZDICT_cover_params_t* coverParams,
ZDICT_fastCover_params_t* fastCoverParams, int optimize);
#endif
programs/fileio.c
+60 -5
View File
@@ -30,6 +30,10 @@
#include <stdlib.h> /* malloc, free */
#include <string.h> /* strcmp, strlen */
#include <errno.h> /* errno */
#include <signal.h>
#ifndef _WIN32
#include <execinfo.h> /* backtrace, backtrace_symbols */
#endif
#if defined (_MSC_VER)
# include <sys/stat.h>
@@ -125,8 +129,6 @@ static UTIL_time_t g_displayClock = UTIL_TIME_INITIALIZER;
/*-************************************
* Signal (Ctrl-C trapping)
**************************************/
#include <signal.h>
static const char* g_artefact = NULL;
static void INThandler(int sig)
{
@@ -158,7 +160,60 @@ static void clearHandler(void)
}
/* ************************************************************
/*-*********************************************************
* Termination signal trapping (Print debug stack trace)
***********************************************************/
#define MAX_STACK_FRAMES 50
#ifndef _WIN32
static void ABRThandler(int sig) {
const char* name;
void* addrlist[MAX_STACK_FRAMES];
char** symbollist;
U32 addrlen, i;
switch (sig) {
case SIGABRT: name = "SIGABRT"; break;
case SIGFPE: name = "SIGFPE"; break;
case SIGILL: name = "SIGILL"; break;
case SIGINT: name = "SIGINT"; break;
case SIGSEGV: name = "SIGSEGV"; break;
default: name = "UNKNOWN";
}
DISPLAY("Caught %s signal, printing stack:\n", name);
/* Retrieve current stack addresses. */
addrlen = backtrace(addrlist, MAX_STACK_FRAMES);
if (addrlen == 0) {
DISPLAY("\n");
return;
}
/* Create readable strings to each frame. */
symbollist = backtrace_symbols(addrlist, addrlen);
/* Print the stack trace, excluding calls handling the signal. */
for (i = ZSTD_START_SYMBOLLIST_FRAME; i < addrlen; i++) {
DISPLAY("%s\n", symbollist[i]);
}
free(symbollist);
/* Reset and raise the signal so default handler runs. */
signal(sig, SIG_DFL);
raise(sig);
}
#endif
void FIO_addAbortHandler()
{
#ifndef _WIN32
signal(SIGABRT, ABRThandler);
signal(SIGFPE, ABRThandler);
signal(SIGILL, ABRThandler);
signal(SIGSEGV, ABRThandler);
signal(SIGBUS, ABRThandler);
#endif
}
/*-************************************************************
* Avoid fseek()'s 2GiB barrier with MSVC, macOS, *BSD, MinGW
***************************************************************/
#if defined(_MSC_VER) && _MSC_VER >= 1400
@@ -1119,8 +1174,8 @@ int FIO_compressMultipleFilenames(const char** inFileNamesTable, unsigned nbFile
if (!dstFileName) {
EXM_THROW(30, "zstd: %s", strerror(errno));
} }
strcpy(dstFileName, inFileNamesTable[u]);
strcat(dstFileName, suffix);
strncpy(dstFileName, inFileNamesTable[u], ifnSize+1 /* Include null */);
strncat(dstFileName, suffix, suffixSize);
missed_files += FIO_compressFilename_dstFile(ress, dstFileName, inFileNamesTable[u], compressionLevel);
} }
programs/fileio.h
+3
View File
@@ -96,6 +96,9 @@ int FIO_decompressMultipleFilenames(const char** srcNamesTable, unsigned nbFiles
const char* dictFileName);
/* custom crash signal handler */
void FIO_addAbortHandler(void);
#if defined (__cplusplus)
}
#endif
programs/platform.h
+11
View File
@@ -148,6 +148,17 @@ static __inline int IS_CONSOLE(FILE* stdStream) {
#endif
#ifndef ZSTD_START_SYMBOLLIST_FRAME
# ifdef __linux__
# define ZSTD_START_SYMBOLLIST_FRAME 2
# elif defined __APPLE__
# define ZSTD_START_SYMBOLLIST_FRAME 4
# else
# define ZSTD_START_SYMBOLLIST_FRAME 0
# endif
#endif
#if defined (__cplusplus)
}
#endif
programs/util.h
+11 -5
View File
@@ -319,15 +319,18 @@ UTIL_STATIC U32 UTIL_isDirectory(const char* infilename)
UTIL_STATIC U32 UTIL_isLink(const char* infilename)
{
#if defined(_WIN32)
/* no symlinks on windows */
(void)infilename;
#else
/* macro guards, as defined in : https://linux.die.net/man/2/lstat */
#if defined(_BSD_SOURCE) \
|| (defined(_XOPEN_SOURCE) && (_XOPEN_SOURCE >= 500)) \
|| (defined(_XOPEN_SOURCE) && defined(_XOPEN_SOURCE_EXTENDED)) \
|| (defined(_POSIX_C_SOURCE) && (_POSIX_C_SOURCE >= 200112L)) \
|| (defined(__APPLE__) && defined(__MACH__))
int r;
stat_t statbuf;
r = lstat(infilename, &statbuf);
if (!r && S_ISLNK(statbuf.st_mode)) return 1;
#endif
(void)infilename;
return 0;
}
@@ -526,7 +529,10 @@ UTIL_STATIC int UTIL_prepareFileList(const char *dirName, char** bufStart, size_
* After finishing usage of the list the structures should be freed with UTIL_freeFileList(params: return value, allocatedBuffer)
* In case of error UTIL_createFileList returns NULL and UTIL_freeFileList should not be called.
*/
UTIL_STATIC const char** UTIL_createFileList(const char **inputNames, unsigned inputNamesNb, char** allocatedBuffer, unsigned* allocatedNamesNb, int followLinks)
UTIL_STATIC const char**
UTIL_createFileList(const char **inputNames, unsigned inputNamesNb,
char** allocatedBuffer, unsigned* allocatedNamesNb,
int followLinks)
{
size_t pos;
unsigned i, nbFiles;
programs/zstd.1
+20 -1
View File
@@ -194,7 +194,7 @@ All arguments after \fB\-\-\fR are treated as files
Use FILEs as training set to create a dictionary\. The training set should contain a lot of small files (> 100), and weight typically 100x the target dictionary size (for example, 10 MB for a 100 KB dictionary)\.
.
.IP
Supports multithreading if \fBzstd\fR is compiled with threading support\. Additional parameters can be specified with \fB\-\-train\-cover\fR\. The legacy dictionary builder can be accessed with \fB\-\-train\-legacy\fR\. Equivalent to \fB\-\-train\-cover=d=8,steps=4\fR\.
Supports multithreading if \fBzstd\fR is compiled with threading support\. Additional parameters can be specified with \fB\-\-train\-fastcover\fR\. The legacy dictionary builder can be accessed with \fB\-\-train\-legacy\fR\. The cover dictionary builder can be accessed with \fB\-\-train\-cover\fR\. Equivalent to \fB\-\-train\-fastCover=d=8,steps=4\fR\.
.
.TP
\fB\-o file\fR
@@ -240,6 +240,25 @@ Examples:
\fBzstd \-\-train\-cover=k=50 FILEs\fR
.
.TP
\fB\-\-train\-fastcover[=k#,d=#,f=#,steps=#,split=#,accel=#]\fR
Same as cover but with extra parameters \fIf\fR and \fIaccel\fR and different default value of split
.
.IP
If \fIsplit\fR is not specified, then it tries \fIsplit\fR = 75. If \fIf\fR is not specified, then it tries \fIf\fR = 20. Requires that 0 < \fIf\fR < 32. If \fIaccel\fR is not specified, then it tries \fIaccel\fR = 1. Requires that 0 < \fIaccel\fR <= 10. Requires that \fId\fR = 6 or \fId\fR = 8.
.
.IP
\fIf\fR is log of size of array that keeps track of frequency of subsegments of size \fId\fR. The subsegment is hashed to an index in the range [0,2^\fIf\fR - 1]. It is possible that 2 different subsegments are hashed to the same index, and they are considered as the same subsegment when computing frequency. Using a higher \fIf\fR reduces collision but takes longer.
.
.IP
Examples:
.
.IP
\fBzstd \-\-train\-fastcover FILEs\fR
.
.IP
\fBzstd \-\-train\-fastcover=d=8,f=15,accel=2 FILEs\fR
.
.TP
\fB\-\-train\-legacy[=selectivity=#]\fR
Use legacy dictionary builder algorithm with the given dictionary \fIselectivity\fR (default: 9)\. The smaller the \fIselectivity\fR value, the denser the dictionary, improving its efficiency but reducing its possible maximum size\. \fB\-\-train\-legacy=s=#\fR is also accepted\.
.
programs/zstd.1.md
+23 -2
View File
@@ -207,9 +207,10 @@ Compression of small files similar to the sample set will be greatly improved.
(for example, 10 MB for a 100 KB dictionary).
Supports multithreading if `zstd` is compiled with threading support.
Additional parameters can be specified with `--train-cover`.
Additional parameters can be specified with `--train-fastcover`.
The legacy dictionary builder can be accessed with `--train-legacy`.
Equivalent to `--train-cover=d=8,steps=4`.
The cover dictionary builder can be accessed with `--train-cover`.
Equivalent to `--train-fastcover=d=8,steps=4`.
* `-o file`:
Dictionary saved into `file` (default name: dictionary).
* `--maxdict=#`:
@@ -261,6 +262,26 @@ Compression of small files similar to the sample set will be greatly improved.
`zstd --train-cover=k=50,split=60 FILEs`
* `--train-fastcover[=k#,d=#,f=#,steps=#,split=#,accel=#]`:
Same as cover but with extra parameters _f_ and _accel_ and different default value of split
If _split_ is not specified, then it tries _split_ = 75.
If _f_ is not specified, then it tries _f_ = 20.
Requires that 0 < _f_ < 32.
If _accel_ is not specified, then it tries _accel_ = 1.
Requires that 0 < _accel_ <= 10.
Requires that _d_ = 6 or _d_ = 8.
_f_ is log of size of array that keeps track of frequency of subsegments of size _d_.
The subsegment is hashed to an index in the range [0,2^_f_ - 1].
It is possible that 2 different subsegments are hashed to the same index, and they are considered as the same subsegment when computing frequency.
Using a higher _f_ reduces collision but takes longer.
Examples:
`zstd --train-fastcover FILEs`
`zstd --train-fastcover=d=8,f=15,accel=2 FILEs`
* `--train-legacy[=selectivity=#]`:
Use legacy dictionary builder algorithm with the given dictionary
_selectivity_ (default: 9).
programs/zstdcli.c
+73 -10
View File
@@ -84,7 +84,10 @@ static U32 g_ldmMinMatch = 0;
static U32 g_ldmHashEveryLog = LDM_PARAM_DEFAULT;
static U32 g_ldmBucketSizeLog = LDM_PARAM_DEFAULT;
#define DEFAULT_SPLITPOINT 1.0
#define DEFAULT_ACCEL 1
typedef enum { cover, fastCover, legacy } dictType;
/*-************************************
* Display Macros
@@ -173,6 +176,7 @@ static int usage_advanced(const char* programName)
DISPLAY( "Dictionary builder : \n");
DISPLAY( "--train ## : create a dictionary from a training set of files \n");
DISPLAY( "--train-cover[=k=#,d=#,steps=#,split=#] : use the cover algorithm with optional args\n");
DISPLAY( "--train-fastcover[=k=#,d=#,f=#,steps=#,split=#,accel=#] : use the fast cover algorithm with optional args\n");
DISPLAY( "--train-legacy[=s=#] : use the legacy algorithm with selectivity (default: %u)\n", g_defaultSelectivityLevel);
DISPLAY( " -o file : `file` is dictionary name (default: %s) \n", g_defaultDictName);
DISPLAY( "--maxdict=# : limit dictionary to specified size (default: %u) \n", g_defaultMaxDictSize);
@@ -296,6 +300,33 @@ static unsigned parseCoverParameters(const char* stringPtr, ZDICT_cover_params_t
return 1;
}
/**
* parseFastCoverParameters() :
* reads fastcover parameters from *stringPtr (e.g. "--train-fastcover=k=48,d=8,f=20,steps=32,accel=2") into *params
* @return 1 means that fastcover parameters were correct
* @return 0 in case of malformed parameters
*/
static unsigned parseFastCoverParameters(const char* stringPtr, ZDICT_fastCover_params_t* params)
{
memset(params, 0, sizeof(*params));
for (; ;) {
if (longCommandWArg(&stringPtr, "k=")) { params->k = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
if (longCommandWArg(&stringPtr, "d=")) { params->d = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
if (longCommandWArg(&stringPtr, "f=")) { params->f = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
if (longCommandWArg(&stringPtr, "steps=")) { params->steps = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
if (longCommandWArg(&stringPtr, "accel=")) { params->accel = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
if (longCommandWArg(&stringPtr, "split=")) {
unsigned splitPercentage = readU32FromChar(&stringPtr);
params->splitPoint = (double)splitPercentage / 100.0;
if (stringPtr[0]==',') { stringPtr++; continue; } else break;
}
return 0;
}
if (stringPtr[0] != 0) return 0;
DISPLAYLEVEL(4, "cover: k=%u\nd=%u\nf=%u\nsteps=%u\nsplit=%u\naccel=%u\n", params->k, params->d, params->f, params->steps, (unsigned)(params->splitPoint * 100), params->accel);
return 1;
}
/**
* parseLegacyParameters() :
* reads legacy dictioanry builter parameters from *stringPtr (e.g. "--train-legacy=selectivity=8") into *selectivity
@@ -317,7 +348,19 @@ static ZDICT_cover_params_t defaultCoverParams(void)
memset(&params, 0, sizeof(params));
params.d = 8;
params.steps = 4;
params.splitPoint = DEFAULT_SPLITPOINT;
params.splitPoint = 1.0;
return params;
}
static ZDICT_fastCover_params_t defaultFastCoverParams(void)
{
ZDICT_fastCover_params_t params;
memset(&params, 0, sizeof(params));
params.d = 8;
params.f = 20;
params.steps = 4;
params.splitPoint = 0.75; /* different from default splitPoint of cover */
params.accel = DEFAULT_ACCEL;
return params;
}
#endif
@@ -433,7 +476,8 @@ int main(int argCount, const char* argv[])
#endif
#ifndef ZSTD_NODICT
ZDICT_cover_params_t coverParams = defaultCoverParams();
int cover = 1;
ZDICT_fastCover_params_t fastCoverParams = defaultFastCoverParams();
dictType dict = fastCover;
#endif
#ifndef ZSTD_NOBENCH
BMK_advancedParams_t benchParams = BMK_initAdvancedParams();
@@ -469,6 +513,9 @@ int main(int argCount, const char* argv[])
if (exeNameMatch(programName, ZSTD_UNLZ4)) { operation=zom_decompress; FIO_setCompressionType(FIO_lz4Compression); } /* behave like unlz4, also supports multiple formats */
memset(&compressionParams, 0, sizeof(compressionParams));
/* init crash handler */
FIO_addAbortHandler();
/* command switches */
for (argNb=1; argNb<argCount; argNb++) {
const char* argument = argv[argNb];
@@ -533,18 +580,29 @@ int main(int argCount, const char* argv[])
operation = zom_train;
if (outFileName == NULL)
outFileName = g_defaultDictName;
cover = 1;
dict = cover;
/* Allow optional arguments following an = */
if (*argument == 0) { memset(&coverParams, 0, sizeof(coverParams)); }
else if (*argument++ != '=') { CLEAN_RETURN(badusage(programName)); }
else if (!parseCoverParameters(argument, &coverParams)) { CLEAN_RETURN(badusage(programName)); }
continue;
}
if (longCommandWArg(&argument, "--train-fastcover")) {
operation = zom_train;
if (outFileName == NULL)
outFileName = g_defaultDictName;
dict = fastCover;
/* Allow optional arguments following an = */
if (*argument == 0) { memset(&fastCoverParams, 0, sizeof(fastCoverParams)); }
else if (*argument++ != '=') { CLEAN_RETURN(badusage(programName)); }
else if (!parseFastCoverParameters(argument, &fastCoverParams)) { CLEAN_RETURN(badusage(programName)); }
continue;
}
if (longCommandWArg(&argument, "--train-legacy")) {
operation = zom_train;
if (outFileName == NULL)
outFileName = g_defaultDictName;
cover = 0;
dict = legacy;
/* Allow optional arguments following an = */
if (*argument == 0) { continue; }
else if (*argument++ != '=') { CLEAN_RETURN(badusage(programName)); }
@@ -849,13 +907,13 @@ int main(int argCount, const char* argv[])
if (cLevelLast > ZSTD_maxCLevel()) cLevelLast = ZSTD_maxCLevel();
if (cLevelLast < cLevel) cLevelLast = cLevel;
if (cLevelLast > cLevel)
DISPLAYLEVEL(2, "Benchmarking levels from %d to %d\n", cLevel, cLevelLast);
DISPLAYLEVEL(3, "Benchmarking levels from %d to %d\n", cLevel, cLevelLast);
if(filenameIdx) {
if(separateFiles) {
unsigned i;
for(i = 0; i < filenameIdx; i++) {
int c;
DISPLAYLEVEL(2, "Benchmarking %s \n", filenameTable[i]);
DISPLAYLEVEL(3, "Benchmarking %s \n", filenameTable[i]);
for(c = cLevel; c <= cLevelLast; c++) {
BMK_benchFilesAdvanced(&filenameTable[i], 1, dictFileName, c, &compressionParams, g_displayLevel, &benchParams);
}
@@ -884,17 +942,22 @@ int main(int argCount, const char* argv[])
zParams.compressionLevel = dictCLevel;
zParams.notificationLevel = g_displayLevel;
zParams.dictID = dictID;
if (cover) {
if (dict == cover) {
int const optimize = !coverParams.k || !coverParams.d;
coverParams.nbThreads = nbWorkers;
coverParams.zParams = zParams;
operationResult = DiB_trainFromFiles(outFileName, maxDictSize, filenameTable, filenameIdx, blockSize, NULL, &coverParams, optimize);
operationResult = DiB_trainFromFiles(outFileName, maxDictSize, filenameTable, filenameIdx, blockSize, NULL, &coverParams, NULL, optimize);
} else if (dict == fastCover) {
int const optimize = !fastCoverParams.k || !fastCoverParams.d;
fastCoverParams.nbThreads = nbWorkers;
fastCoverParams.zParams = zParams;
operationResult = DiB_trainFromFiles(outFileName, maxDictSize, filenameTable, filenameIdx, blockSize, NULL, NULL, &fastCoverParams, optimize);
} else {
ZDICT_legacy_params_t dictParams;
memset(&dictParams, 0, sizeof(dictParams));
dictParams.selectivityLevel = dictSelect;
dictParams.zParams = zParams;
operationResult = DiB_trainFromFiles(outFileName, maxDictSize, filenameTable, filenameIdx, blockSize, &dictParams, NULL, 0);
operationResult = DiB_trainFromFiles(outFileName, maxDictSize, filenameTable, filenameIdx, blockSize, &dictParams, NULL, NULL, 0);
}
#endif
goto _end;