Merge branch 'dev' into adapt
This commit is contained in:
Yann Collet
@@ -114,7 +114,7 @@ clean:
|
||||
ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD DragonFly NetBSD MSYS_NT))
|
||||
|
||||
HOST_OS = POSIX
|
||||
CMAKE_PARAMS = -DZSTD_BUILD_CONTRIB:BOOL=ON -DZSTD_BUILD_STATIC:BOOL=ON -DZSTD_BUILD_TESTS:BOOL=ON -DZSTD_ZLIB_SUPPORT:BOOL=ON -DZSTD_LZMA_SUPPORT:BOOL=ON
|
||||
CMAKE_PARAMS = -DZSTD_BUILD_CONTRIB:BOOL=ON -DZSTD_BUILD_STATIC:BOOL=ON -DZSTD_BUILD_TESTS:BOOL=ON -DZSTD_ZLIB_SUPPORT:BOOL=ON -DZSTD_LZMA_SUPPORT:BOOL=ON -DCMAKE_BUILD_TYPE=Release
|
||||
|
||||
.PHONY: list
|
||||
list:
|
||||
|
||||
@@ -121,6 +121,8 @@ A `cmake` project generator is provided within `build/cmake`.
|
||||
It can generate Makefiles or other build scripts
|
||||
to create `zstd` binary, and `libzstd` dynamic and static libraries.
|
||||
|
||||
By default, `CMAKE_BUILD_TYPE` is set to `Release`.
|
||||
|
||||
#### Meson
|
||||
|
||||
A Meson project is provided within `contrib/meson`.
|
||||
|
||||
@@ -18,3 +18,14 @@ Studio*
|
||||
|
||||
# CMake
|
||||
cmake/build/
|
||||
CMakeCache.txt
|
||||
CMakeFiles
|
||||
CMakeScripts
|
||||
Testing
|
||||
Makefile
|
||||
cmake_install.cmake
|
||||
install_manifest.txt
|
||||
compile_commands.json
|
||||
CTestTestfile.cmake
|
||||
build
|
||||
lib
|
||||
|
||||
@@ -10,6 +10,10 @@
|
||||
PROJECT(zstd)
|
||||
CMAKE_MINIMUM_REQUIRED(VERSION 2.8.9)
|
||||
SET(ZSTD_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../..")
|
||||
|
||||
# Ensure Release build even if not invoked via Makefile
|
||||
SET(CMAKE_BUILD_TYPE "Release")
|
||||
|
||||
LIST(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/CMakeModules")
|
||||
INCLUDE(GNUInstallDirs)
|
||||
|
||||
|
||||
+124
-114
@@ -171,6 +171,8 @@ struct BMK_timeState_t{
|
||||
static void BMK_initCCtx(ZSTD_CCtx* ctx,
|
||||
const void* dictBuffer, size_t dictBufferSize, int cLevel,
|
||||
const ZSTD_compressionParameters* comprParams, const BMK_advancedParams_t* adv) {
|
||||
ZSTD_CCtx_reset(ctx);
|
||||
ZSTD_CCtx_resetParameters(ctx);
|
||||
if (adv->nbWorkers==1) {
|
||||
ZSTD_CCtx_setParameter(ctx, ZSTD_p_nbWorkers, 0);
|
||||
} else {
|
||||
@@ -195,6 +197,7 @@ static void BMK_initCCtx(ZSTD_CCtx* ctx,
|
||||
|
||||
static void BMK_initDCtx(ZSTD_DCtx* dctx,
|
||||
const void* dictBuffer, size_t dictBufferSize) {
|
||||
ZSTD_DCtx_reset(dctx);
|
||||
ZSTD_DCtx_loadDictionary(dctx, dictBuffer, dictBufferSize);
|
||||
}
|
||||
|
||||
@@ -291,34 +294,34 @@ BMK_customReturn_t BMK_benchFunction(
|
||||
BMK_initFn_t initFn, void* initPayload,
|
||||
size_t blockCount,
|
||||
const void* const * const srcBlockBuffers, const size_t* srcBlockSizes,
|
||||
void* const * const dstBlockBuffers, const size_t* dstBlockCapacities,
|
||||
void* const * const dstBlockBuffers, const size_t* dstBlockCapacities, size_t* blockResult,
|
||||
unsigned nbLoops) {
|
||||
size_t srcSize = 0, dstSize = 0, ind = 0;
|
||||
size_t dstSize = 0;
|
||||
U64 totalTime;
|
||||
|
||||
BMK_customReturn_t retval;
|
||||
UTIL_time_t clockStart;
|
||||
|
||||
{
|
||||
unsigned i;
|
||||
for(i = 0; i < blockCount; i++) {
|
||||
memset(dstBlockBuffers[i], 0xE5, dstBlockCapacities[i]); /* warm up and erase result buffer */
|
||||
}
|
||||
|
||||
UTIL_sleepMilli(5); /* give processor time to other processes */
|
||||
UTIL_waitForNextTick();
|
||||
}
|
||||
|
||||
if(!nbLoops) {
|
||||
EXM_THROW_ND(1, BMK_customReturn_t, "nbLoops must be nonzero \n");
|
||||
}
|
||||
|
||||
for(ind = 0; ind < blockCount; ind++) {
|
||||
srcSize += srcBlockSizes[ind];
|
||||
{
|
||||
size_t i;
|
||||
for(i = 0; i < blockCount; i++) {
|
||||
memset(dstBlockBuffers[i], 0xE5, dstBlockCapacities[i]); /* warm up and erase result buffer */
|
||||
}
|
||||
#if 0
|
||||
/* based on testing these seem to lower accuracy of multiple calls of 1 nbLoops vs 1 call of multiple nbLoops
|
||||
* (Makes former slower)
|
||||
*/
|
||||
UTIL_sleepMilli(5); /* give processor time to other processes */
|
||||
UTIL_waitForNextTick();
|
||||
#endif
|
||||
}
|
||||
|
||||
{
|
||||
unsigned i, j, firstIter = 1;
|
||||
unsigned i, j;
|
||||
clockStart = UTIL_getTime();
|
||||
if(initFn != NULL) { initFn(initPayload); }
|
||||
for(i = 0; i < nbLoops; i++) {
|
||||
@@ -327,11 +330,13 @@ BMK_customReturn_t BMK_benchFunction(
|
||||
if(ZSTD_isError(res)) {
|
||||
EXM_THROW_ND(2, BMK_customReturn_t, "Function benchmarking failed on block %u of size %u : %s \n",
|
||||
j, (U32)dstBlockCapacities[j], ZSTD_getErrorName(res));
|
||||
} else if(firstIter) {
|
||||
} else if(i == nbLoops - 1) {
|
||||
dstSize += res;
|
||||
}
|
||||
if(blockResult != NULL) {
|
||||
blockResult[j] = res;
|
||||
}
|
||||
}
|
||||
}
|
||||
firstIter = 0;
|
||||
}
|
||||
totalTime = UTIL_clockSpanNano(clockStart);
|
||||
}
|
||||
@@ -353,6 +358,9 @@ void BMK_resetTimeState(BMK_timedFnState_t* r, unsigned nbSeconds) {
|
||||
|
||||
BMK_timedFnState_t* BMK_createTimeState(unsigned nbSeconds) {
|
||||
BMK_timedFnState_t* r = (BMK_timedFnState_t*)malloc(sizeof(struct BMK_timeState_t));
|
||||
if(r == NULL) {
|
||||
return r;
|
||||
}
|
||||
BMK_resetTimeState(r, nbSeconds);
|
||||
return r;
|
||||
}
|
||||
@@ -361,13 +369,14 @@ void BMK_freeTimeState(BMK_timedFnState_t* state) {
|
||||
free(state);
|
||||
}
|
||||
|
||||
/* make option for dstBlocks to be */
|
||||
BMK_customTimedReturn_t BMK_benchFunctionTimed(
|
||||
BMK_timedFnState_t* cont,
|
||||
BMK_benchFn_t benchFn, void* benchPayload,
|
||||
BMK_initFn_t initFn, void* initPayload,
|
||||
size_t blockCount,
|
||||
const void* const* const srcBlockBuffers, const size_t* srcBlockSizes,
|
||||
void* const* const dstBlockBuffers, const size_t* dstBlockCapacities)
|
||||
void * const * const dstBlockBuffers, const size_t * dstBlockCapacities, size_t* blockResults)
|
||||
{
|
||||
U64 fastest = cont->fastestTime;
|
||||
int completed = 0;
|
||||
@@ -382,9 +391,9 @@ BMK_customTimedReturn_t BMK_benchFunctionTimed(
|
||||
UTIL_sleep(COOLPERIOD_SEC);
|
||||
cont->coolTime = UTIL_getTime();
|
||||
}
|
||||
|
||||
/* reinitialize capacity */
|
||||
r.result = BMK_benchFunction(benchFn, benchPayload, initFn, initPayload,
|
||||
blockCount, srcBlockBuffers, srcBlockSizes, dstBlockBuffers, dstBlockCapacities, cont->nbLoops);
|
||||
blockCount, srcBlockBuffers, srcBlockSizes, dstBlockBuffers, dstBlockCapacities, blockResults, cont->nbLoops);
|
||||
if(r.result.error) { /* completed w/ error */
|
||||
r.completed = 1;
|
||||
return r;
|
||||
@@ -393,7 +402,7 @@ BMK_customTimedReturn_t BMK_benchFunctionTimed(
|
||||
{ U64 const loopDuration = r.result.result.nanoSecPerRun * cont->nbLoops;
|
||||
r.completed = (cont->timeRemaining <= loopDuration);
|
||||
cont->timeRemaining -= loopDuration;
|
||||
if (loopDuration > 0) {
|
||||
if (loopDuration > (TIMELOOP_NANOSEC / 100)) {
|
||||
fastest = MIN(fastest, r.result.result.nanoSecPerRun);
|
||||
if(loopDuration >= MINUSABLETIME) {
|
||||
r.result.result.nanoSecPerRun = fastest;
|
||||
@@ -418,9 +427,9 @@ BMK_customTimedReturn_t BMK_benchFunctionTimed(
|
||||
/* benchMem with no allocation */
|
||||
static BMK_return_t BMK_benchMemAdvancedNoAlloc(
|
||||
const void ** const srcPtrs, size_t* const srcSizes,
|
||||
void** const cPtrs, size_t* const cSizes,
|
||||
void** const cPtrs, size_t* const cCapacities, size_t* const cSizes,
|
||||
void** const resPtrs, size_t* const resSizes,
|
||||
void* resultBuffer, void* compressedBuffer,
|
||||
void** resultBufferPtr, void* compressedBuffer,
|
||||
const size_t maxCompressedSize,
|
||||
BMK_timedFnState_t* timeStateCompress, BMK_timedFnState_t* timeStateDecompress,
|
||||
|
||||
@@ -432,7 +441,7 @@ static BMK_return_t BMK_benchMemAdvancedNoAlloc(
|
||||
int displayLevel, const char* displayName, const BMK_advancedParams_t* adv)
|
||||
{
|
||||
size_t const blockSize = ((adv->blockSize>=32 && (adv->mode != BMK_decodeOnly)) ? adv->blockSize : srcSize) + (!srcSize); /* avoid div by 0 */
|
||||
BMK_return_t results;
|
||||
BMK_return_t results = { { 0, 0, 0, 0 }, 0 } ;
|
||||
size_t const loadedCompressedSize = srcSize;
|
||||
size_t cSize = 0;
|
||||
double ratio = 0.;
|
||||
@@ -454,13 +463,13 @@ static BMK_return_t BMK_benchMemAdvancedNoAlloc(
|
||||
srcPtr += fileSizes[fileNb];
|
||||
}
|
||||
{ size_t const decodedSize = (size_t)totalDSize64;
|
||||
free(resultBuffer);
|
||||
resultBuffer = malloc(decodedSize);
|
||||
if (!resultBuffer) {
|
||||
free(*resultBufferPtr);
|
||||
*resultBufferPtr = malloc(decodedSize);
|
||||
if (!(*resultBufferPtr)) {
|
||||
EXM_THROW(33, BMK_return_t, "not enough memory");
|
||||
}
|
||||
if (totalDSize64 > decodedSize) {
|
||||
free(resultBuffer);
|
||||
free(*resultBufferPtr);
|
||||
EXM_THROW(32, BMK_return_t, "original size is too large"); /* size_t overflow */
|
||||
}
|
||||
cSize = srcSize;
|
||||
@@ -472,7 +481,7 @@ static BMK_return_t BMK_benchMemAdvancedNoAlloc(
|
||||
/* Init data blocks */
|
||||
{ const char* srcPtr = (const char*)srcBuffer;
|
||||
char* cPtr = (char*)compressedBuffer;
|
||||
char* resPtr = (char*)resultBuffer;
|
||||
char* resPtr = (char*)(*resultBufferPtr);
|
||||
U32 fileNb;
|
||||
for (nbBlocks=0, fileNb=0; fileNb<nbFiles; fileNb++) {
|
||||
size_t remaining = fileSizes[fileNb];
|
||||
@@ -483,11 +492,11 @@ static BMK_return_t BMK_benchMemAdvancedNoAlloc(
|
||||
srcPtrs[nbBlocks] = (const void*)srcPtr;
|
||||
srcSizes[nbBlocks] = thisBlockSize;
|
||||
cPtrs[nbBlocks] = (void*)cPtr;
|
||||
cSizes[nbBlocks] = (adv->mode == BMK_decodeOnly) ? thisBlockSize : ZSTD_compressBound(thisBlockSize);
|
||||
cCapacities[nbBlocks] = (adv->mode == BMK_decodeOnly) ? thisBlockSize : ZSTD_compressBound(thisBlockSize);
|
||||
resPtrs[nbBlocks] = (void*)resPtr;
|
||||
resSizes[nbBlocks] = (adv->mode == BMK_decodeOnly) ? (size_t) ZSTD_findDecompressedSize(srcPtr, thisBlockSize) : thisBlockSize;
|
||||
srcPtr += thisBlockSize;
|
||||
cPtr += cSizes[nbBlocks];
|
||||
cPtr += cCapacities[nbBlocks];
|
||||
resPtr += thisBlockSize;
|
||||
remaining -= thisBlockSize;
|
||||
}
|
||||
@@ -538,7 +547,7 @@ static BMK_return_t BMK_benchMemAdvancedNoAlloc(
|
||||
while(!(intermediateResultCompress.completed && intermediateResultDecompress.completed)) {
|
||||
if(!intermediateResultCompress.completed) {
|
||||
intermediateResultCompress = BMK_benchFunctionTimed(timeStateCompress, &local_defaultCompress, (void*)ctx, &local_initCCtx, (void*)&cctxprep,
|
||||
nbBlocks, srcPtrs, srcSizes, cPtrs, cSizes);
|
||||
nbBlocks, srcPtrs, srcSizes, cPtrs, cCapacities, cSizes);
|
||||
if(intermediateResultCompress.result.error) {
|
||||
results.error = intermediateResultCompress.result.error;
|
||||
return results;
|
||||
@@ -546,9 +555,7 @@ static BMK_return_t BMK_benchMemAdvancedNoAlloc(
|
||||
ratio = (double)(srcSize / intermediateResultCompress.result.result.sumOfReturn);
|
||||
{
|
||||
int const ratioAccuracy = (ratio < 10.) ? 3 : 2;
|
||||
double const compressionSpeed = ((double)srcSize / intermediateResultCompress.result.result.nanoSecPerRun) * 1000;
|
||||
int const cSpeedAccuracy = (compressionSpeed < 10.) ? 2 : 1;
|
||||
results.result.cSpeed = compressionSpeed * 1000000;
|
||||
results.result.cSpeed = (srcSize * TIMELOOP_NANOSEC / intermediateResultCompress.result.result.nanoSecPerRun);
|
||||
cSize = intermediateResultCompress.result.result.sumOfReturn;
|
||||
results.result.cSize = cSize;
|
||||
ratio = (double)srcSize / results.result.cSize;
|
||||
@@ -556,13 +563,13 @@ static BMK_return_t BMK_benchMemAdvancedNoAlloc(
|
||||
DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.*f),%6.*f MB/s\r",
|
||||
marks[markNb], displayName, (U32)srcSize, (U32)results.result.cSize,
|
||||
ratioAccuracy, ratio,
|
||||
cSpeedAccuracy, compressionSpeed);
|
||||
results.result.cSpeed < (10 MB) ? 2 : 1, (double)results.result.cSpeed / (1 MB));
|
||||
}
|
||||
}
|
||||
|
||||
if(!intermediateResultDecompress.completed) {
|
||||
intermediateResultDecompress = BMK_benchFunctionTimed(timeStateDecompress, &local_defaultDecompress, (void*)(dctx), &local_initDCtx, (void*)&dctxprep,
|
||||
nbBlocks, (const void* const*)cPtrs, cSizes, resPtrs, resSizes);
|
||||
nbBlocks, (const void* const*)cPtrs, cSizes, resPtrs, resSizes, NULL);
|
||||
if(intermediateResultDecompress.result.error) {
|
||||
results.error = intermediateResultDecompress.result.error;
|
||||
return results;
|
||||
@@ -570,79 +577,79 @@ static BMK_return_t BMK_benchMemAdvancedNoAlloc(
|
||||
|
||||
{
|
||||
int const ratioAccuracy = (ratio < 10.) ? 3 : 2;
|
||||
double const compressionSpeed = results.result.cSpeed / 1000000;
|
||||
int const cSpeedAccuracy = (compressionSpeed < 10.) ? 2 : 1;
|
||||
double const decompressionSpeed = ((double)srcSize / intermediateResultDecompress.result.result.nanoSecPerRun) * 1000;
|
||||
results.result.dSpeed = decompressionSpeed * 1000000;
|
||||
results.result.dSpeed = (srcSize * TIMELOOP_NANOSEC/ intermediateResultDecompress.result.result.nanoSecPerRun);
|
||||
markNb = (markNb+1) % NB_MARKS;
|
||||
DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.*f),%6.*f MB/s ,%6.1f MB/s \r",
|
||||
marks[markNb], displayName, (U32)srcSize, (U32)results.result.cSize,
|
||||
ratioAccuracy, ratio,
|
||||
cSpeedAccuracy, compressionSpeed,
|
||||
decompressionSpeed);
|
||||
results.result.cSpeed < (10 MB) ? 2 : 1, (double)results.result.cSpeed / (1 MB),
|
||||
(double)results.result.dSpeed / (1 MB));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
} else {
|
||||
} else { //iterMode;
|
||||
if(adv->mode != BMK_decodeOnly) {
|
||||
|
||||
BMK_customReturn_t compressionResults = BMK_benchFunction(&local_defaultCompress, (void*)ctx, &local_initCCtx, (void*)&cctxprep,
|
||||
nbBlocks, srcPtrs, srcSizes, cPtrs, cSizes, adv->nbSeconds);
|
||||
nbBlocks, srcPtrs, srcSizes, cPtrs, cCapacities, cSizes, adv->nbSeconds);
|
||||
if(compressionResults.error) {
|
||||
results.error = compressionResults.error;
|
||||
return results;
|
||||
}
|
||||
|
||||
if(compressionResults.result.nanoSecPerRun == 0) {
|
||||
results.result.cSpeed = 0;
|
||||
} else {
|
||||
results.result.cSpeed = (double)srcSize / compressionResults.result.nanoSecPerRun * TIMELOOP_NANOSEC;
|
||||
results.result.cSpeed = srcSize * TIMELOOP_NANOSEC / compressionResults.result.nanoSecPerRun;
|
||||
}
|
||||
|
||||
results.result.cSize = compressionResults.result.sumOfReturn;
|
||||
{
|
||||
int const ratioAccuracy = (ratio < 10.) ? 3 : 2;
|
||||
double const compressionSpeed = results.result.cSpeed / 1000000;
|
||||
int const cSpeedAccuracy = (compressionSpeed < 10.) ? 2 : 1;
|
||||
cSize = compressionResults.result.sumOfReturn;
|
||||
results.result.cSize = cSize;
|
||||
ratio = (double)srcSize / results.result.cSize;
|
||||
markNb = (markNb+1) % NB_MARKS;
|
||||
DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.*f),%6.*f MB/s\r",
|
||||
marks[markNb], displayName, (U32)srcSize, (U32)results.result.cSize,
|
||||
ratioAccuracy, ratio,
|
||||
cSpeedAccuracy, compressionSpeed);
|
||||
results.result.cSpeed < (10 MB) ? 2 : 1, (double)results.result.cSpeed / (1 MB));
|
||||
}
|
||||
}
|
||||
if(adv->mode != BMK_compressOnly) {
|
||||
BMK_customReturn_t decompressionResults = BMK_benchFunction(
|
||||
&local_defaultDecompress, (void*)(dctx),
|
||||
&local_initDCtx, (void*)&dctxprep, nbBlocks,
|
||||
(const void* const*)cPtrs, cSizes, resPtrs, resSizes,
|
||||
(const void* const*)cPtrs, cSizes, resPtrs, resSizes, NULL,
|
||||
adv->nbSeconds);
|
||||
if(decompressionResults.error) {
|
||||
results.error = decompressionResults.error;
|
||||
return results;
|
||||
}
|
||||
|
||||
if(decompressionResults.result.nanoSecPerRun == 0) {
|
||||
results.result.dSpeed = 0;
|
||||
} else {
|
||||
results.result.dSpeed = (double)srcSize / decompressionResults.result.nanoSecPerRun * TIMELOOP_NANOSEC;
|
||||
results.result.dSpeed = srcSize * TIMELOOP_NANOSEC / decompressionResults.result.nanoSecPerRun;
|
||||
}
|
||||
{ int const ratioAccuracy = (ratio < 10.) ? 3 : 2;
|
||||
double const compressionSpeed = results.result.cSpeed / 1000000;
|
||||
int const cSpeedAccuracy = (compressionSpeed < 10.) ? 2 : 1;
|
||||
double const decompressionSpeed = ((double)srcSize / decompressionResults.result.nanoSecPerRun) * 1000;
|
||||
results.result.dSpeed = decompressionSpeed * 1000000;
|
||||
|
||||
{
|
||||
int const ratioAccuracy = (ratio < 10.) ? 3 : 2;
|
||||
markNb = (markNb+1) % NB_MARKS;
|
||||
DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.*f),%6.*f MB/s ,%6.1f MB/s \r",
|
||||
marks[markNb], displayName, (U32)srcSize, (U32)results.result.cSize,
|
||||
ratioAccuracy, ratio,
|
||||
cSpeedAccuracy, compressionSpeed,
|
||||
decompressionSpeed);
|
||||
results.result.cSpeed < (10 MB) ? 2 : 1, (double)results.result.cSpeed / (1 MB),
|
||||
(double)results.result.dSpeed / (1 MB));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/* CRC Checking */
|
||||
{ U64 const crcCheck = XXH64(resultBuffer, srcSize, 0);
|
||||
{ void* resultBuffer = *resultBufferPtr;
|
||||
U64 const crcCheck = XXH64(resultBuffer, srcSize, 0);
|
||||
/* adv->mode == 0 -> compress + decompress */
|
||||
if ((adv->mode == BMK_both) && (crcOrig!=crcCheck)) {
|
||||
size_t u;
|
||||
@@ -682,8 +689,8 @@ static BMK_return_t BMK_benchMemAdvancedNoAlloc(
|
||||
} /* CRC Checking */
|
||||
|
||||
if (displayLevel == 1) { /* hidden display mode -q, used by python speed benchmark */
|
||||
double const cSpeed = results.result.cSpeed / 1000000;
|
||||
double const dSpeed = results.result.dSpeed / 1000000;
|
||||
double const cSpeed = (double)results.result.cSpeed / (1 MB);
|
||||
double const dSpeed = (double)results.result.dSpeed / (1 MB);
|
||||
if (adv->additionalParam) {
|
||||
DISPLAY("-%-3i%11i (%5.3f) %6.2f MB/s %6.1f MB/s %s (param=%d)\n", cLevel, (int)cSize, ratio, cSpeed, dSpeed, displayName, adv->additionalParam);
|
||||
} else {
|
||||
@@ -692,14 +699,16 @@ static BMK_return_t BMK_benchMemAdvancedNoAlloc(
|
||||
}
|
||||
DISPLAYLEVEL(2, "%2i#\n", cLevel);
|
||||
} /* Bench */
|
||||
results.result.cMem = (1ULL << (comprParams->windowLog)) + ZSTD_sizeof_CCtx(ctx);
|
||||
results.error = 0;
|
||||
return results;
|
||||
}
|
||||
|
||||
BMK_return_t BMK_benchMemAdvanced(const void* srcBuffer, size_t srcSize,
|
||||
void* dstBuffer, size_t dstCapacity,
|
||||
const size_t* fileSizes, unsigned nbFiles,
|
||||
const int cLevel, const ZSTD_compressionParameters* comprParams,
|
||||
const void* dictBuffer, size_t dictBufferSize,
|
||||
ZSTD_CCtx* ctx, ZSTD_DCtx* dctx,
|
||||
int displayLevel, const char* displayName, const BMK_advancedParams_t* adv)
|
||||
|
||||
{
|
||||
@@ -707,47 +716,72 @@ BMK_return_t BMK_benchMemAdvanced(const void* srcBuffer, size_t srcSize,
|
||||
U32 const maxNbBlocks = (U32) ((srcSize + (blockSize-1)) / blockSize) + nbFiles;
|
||||
|
||||
/* these are the blockTable parameters, just split up */
|
||||
const void ** const srcPtrs = (const void** const)malloc(maxNbBlocks * sizeof(void*));
|
||||
size_t* const srcSizes = (size_t* const)malloc(maxNbBlocks * sizeof(size_t));
|
||||
const void ** const srcPtrs = (const void**)malloc(maxNbBlocks * sizeof(void*));
|
||||
size_t* const srcSizes = (size_t*)malloc(maxNbBlocks * sizeof(size_t));
|
||||
|
||||
void ** const cPtrs = (void** const)malloc(maxNbBlocks * sizeof(void*));
|
||||
size_t* const cSizes = (size_t* const)malloc(maxNbBlocks * sizeof(size_t));
|
||||
|
||||
void ** const resPtrs = (void** const)malloc(maxNbBlocks * sizeof(void*));
|
||||
size_t* const resSizes = (size_t* const)malloc(maxNbBlocks * sizeof(size_t));
|
||||
void ** const cPtrs = (void**)malloc(maxNbBlocks * sizeof(void*));
|
||||
size_t* const cSizes = (size_t*)malloc(maxNbBlocks * sizeof(size_t));
|
||||
size_t* const cCapacities = (size_t*)malloc(maxNbBlocks * sizeof(size_t));
|
||||
|
||||
void ** const resPtrs = (void**)malloc(maxNbBlocks * sizeof(void*));
|
||||
size_t* const resSizes = (size_t*)malloc(maxNbBlocks * sizeof(size_t));
|
||||
|
||||
const size_t maxCompressedSize = ZSTD_compressBound(srcSize) + (maxNbBlocks * 1024); /* add some room for safety */
|
||||
void* compressedBuffer = malloc(maxCompressedSize);
|
||||
void* resultBuffer = malloc(srcSize);
|
||||
BMK_timedFnState_t* timeStateCompress = BMK_createTimeState(adv->nbSeconds);
|
||||
BMK_timedFnState_t* timeStateDecompress = BMK_createTimeState(adv->nbSeconds);
|
||||
|
||||
BMK_return_t results;
|
||||
int allocationincomplete = !compressedBuffer || !resultBuffer ||
|
||||
!srcPtrs || !srcSizes || !cPtrs || !cSizes || !resPtrs || !resSizes;
|
||||
if (!allocationincomplete) {
|
||||
results = BMK_benchMemAdvancedNoAlloc(srcPtrs, srcSizes, cPtrs, cSizes,
|
||||
resPtrs, resSizes, resultBuffer, compressedBuffer, maxCompressedSize, timeStateCompress, timeStateDecompress,
|
||||
ZSTD_CCtx* ctx = ZSTD_createCCtx();
|
||||
ZSTD_DCtx* dctx = ZSTD_createDCtx();
|
||||
|
||||
const size_t maxCompressedSize = dstCapacity ? dstCapacity : ZSTD_compressBound(srcSize) + (maxNbBlocks * 1024);
|
||||
|
||||
void* const internalDstBuffer = dstBuffer ? NULL : malloc(maxCompressedSize);
|
||||
void* const compressedBuffer = dstBuffer ? dstBuffer : internalDstBuffer;
|
||||
|
||||
BMK_return_t results = { { 0, 0, 0, 0 }, 0 };
|
||||
|
||||
int parametersConflict = !dstBuffer ^ !dstCapacity;
|
||||
|
||||
void* resultBuffer = srcSize ? malloc(srcSize) : NULL;
|
||||
|
||||
int allocationincomplete = !srcPtrs || !srcSizes || !cPtrs ||
|
||||
!cSizes || !cCapacities || !resPtrs || !resSizes ||
|
||||
!timeStateCompress || !timeStateDecompress || !compressedBuffer || !resultBuffer;
|
||||
|
||||
|
||||
|
||||
if (!allocationincomplete && !parametersConflict) {
|
||||
results = BMK_benchMemAdvancedNoAlloc(srcPtrs, srcSizes, cPtrs, cCapacities, cSizes,
|
||||
resPtrs, resSizes, &resultBuffer, compressedBuffer, maxCompressedSize, timeStateCompress, timeStateDecompress,
|
||||
srcBuffer, srcSize, fileSizes, nbFiles, cLevel, comprParams,
|
||||
dictBuffer, dictBufferSize, ctx, dctx, displayLevel, displayName, adv);
|
||||
}
|
||||
|
||||
/* clean up */
|
||||
BMK_freeTimeState(timeStateCompress);
|
||||
BMK_freeTimeState(timeStateDecompress);
|
||||
free(compressedBuffer);
|
||||
|
||||
ZSTD_freeCCtx(ctx);
|
||||
ZSTD_freeDCtx(dctx);
|
||||
|
||||
free(internalDstBuffer);
|
||||
free(resultBuffer);
|
||||
|
||||
free((void*)srcPtrs);
|
||||
free(srcSizes);
|
||||
free(cPtrs);
|
||||
free(cSizes);
|
||||
free(cCapacities);
|
||||
free(resPtrs);
|
||||
free(resSizes);
|
||||
|
||||
if(allocationincomplete) {
|
||||
EXM_THROW(31, BMK_return_t, "allocation error : not enough memory");
|
||||
}
|
||||
results.error = 0;
|
||||
|
||||
if(parametersConflict) {
|
||||
EXM_THROW(32, BMK_return_t, "Conflicting input results");
|
||||
}
|
||||
return results;
|
||||
}
|
||||
|
||||
@@ -755,42 +789,17 @@ BMK_return_t BMK_benchMem(const void* srcBuffer, size_t srcSize,
|
||||
const size_t* fileSizes, unsigned nbFiles,
|
||||
const int cLevel, const ZSTD_compressionParameters* comprParams,
|
||||
const void* dictBuffer, size_t dictBufferSize,
|
||||
ZSTD_CCtx* ctx, ZSTD_DCtx* dctx,
|
||||
int displayLevel, const char* displayName) {
|
||||
|
||||
const BMK_advancedParams_t adv = BMK_initAdvancedParams();
|
||||
return BMK_benchMemAdvanced(srcBuffer, srcSize,
|
||||
NULL, 0,
|
||||
fileSizes, nbFiles,
|
||||
cLevel, comprParams,
|
||||
dictBuffer, dictBufferSize,
|
||||
ctx, dctx,
|
||||
displayLevel, displayName, &adv);
|
||||
}
|
||||
|
||||
static BMK_return_t BMK_benchMemCtxless(const void* srcBuffer, size_t srcSize,
|
||||
const size_t* fileSizes, unsigned nbFiles,
|
||||
int cLevel, const ZSTD_compressionParameters* const comprParams,
|
||||
const void* dictBuffer, size_t dictBufferSize,
|
||||
int displayLevel, const char* displayName,
|
||||
const BMK_advancedParams_t* const adv)
|
||||
{
|
||||
BMK_return_t res;
|
||||
ZSTD_CCtx* ctx = ZSTD_createCCtx();
|
||||
ZSTD_DCtx* dctx = ZSTD_createDCtx();
|
||||
if(ctx == NULL || dctx == NULL) {
|
||||
EXM_THROW(12, BMK_return_t, "not enough memory for contexts");
|
||||
}
|
||||
res = BMK_benchMemAdvanced(srcBuffer, srcSize,
|
||||
fileSizes, nbFiles,
|
||||
cLevel, comprParams,
|
||||
dictBuffer, dictBufferSize,
|
||||
ctx, dctx,
|
||||
displayLevel, displayName, adv);
|
||||
ZSTD_freeCCtx(ctx);
|
||||
ZSTD_freeDCtx(dctx);
|
||||
return res;
|
||||
}
|
||||
|
||||
static size_t BMK_findMaxMem(U64 requiredMem)
|
||||
{
|
||||
size_t const step = 64 MB;
|
||||
@@ -803,7 +812,7 @@ static size_t BMK_findMaxMem(U64 requiredMem)
|
||||
do {
|
||||
testmem = (BYTE*)malloc((size_t)requiredMem);
|
||||
requiredMem -= step;
|
||||
} while (!testmem);
|
||||
} while (!testmem && requiredMem > 0);
|
||||
|
||||
free(testmem);
|
||||
return (size_t)(requiredMem);
|
||||
@@ -831,12 +840,12 @@ static BMK_return_t BMK_benchCLevel(const void* srcBuffer, size_t benchedSize,
|
||||
if (displayLevel == 1 && !adv->additionalParam)
|
||||
DISPLAY("bench %s %s: input %u bytes, %u seconds, %u KB blocks\n", ZSTD_VERSION_STRING, ZSTD_GIT_COMMIT_STRING, (U32)benchedSize, adv->nbSeconds, (U32)(adv->blockSize>>10));
|
||||
|
||||
res = BMK_benchMemCtxless(srcBuffer, benchedSize,
|
||||
fileSizes, nbFiles,
|
||||
cLevel, comprParams,
|
||||
dictBuffer, dictBufferSize,
|
||||
displayLevel, displayName,
|
||||
adv);
|
||||
res = BMK_benchMemAdvanced(srcBuffer, benchedSize,
|
||||
NULL, 0,
|
||||
fileSizes, nbFiles,
|
||||
cLevel, comprParams,
|
||||
dictBuffer, dictBufferSize,
|
||||
displayLevel, displayName, adv);
|
||||
|
||||
return res;
|
||||
}
|
||||
@@ -931,7 +940,8 @@ BMK_return_t BMK_benchFilesAdvanced(const char* const * const fileNamesTable, un
|
||||
if ((U64)benchedSize > totalSizeToLoad) benchedSize = (size_t)totalSizeToLoad;
|
||||
if (benchedSize < totalSizeToLoad)
|
||||
DISPLAY("Not enough memory; testing %u MB only...\n", (U32)(benchedSize >> 20));
|
||||
srcBuffer = malloc(benchedSize);
|
||||
|
||||
srcBuffer = benchedSize ? malloc(benchedSize) : NULL;
|
||||
if (!srcBuffer) {
|
||||
free(dictBuffer);
|
||||
free(fileSizes);
|
||||
|
||||
+14
-12
@@ -32,8 +32,9 @@ extern "C" {
|
||||
|
||||
typedef struct {
|
||||
size_t cSize;
|
||||
double cSpeed; /* bytes / sec */
|
||||
double dSpeed;
|
||||
U64 cSpeed; /* bytes / sec */
|
||||
U64 dSpeed;
|
||||
size_t cMem;
|
||||
} BMK_result_t;
|
||||
|
||||
ERROR_STRUCT(BMK_result_t, BMK_return_t);
|
||||
@@ -125,8 +126,6 @@ BMK_return_t BMK_syntheticTest(int cLevel, double compressibility,
|
||||
* comprParams - basic compression parameters
|
||||
* dictBuffer - a dictionary if used, null otherwise
|
||||
* dictBufferSize - size of dictBuffer, 0 otherwise
|
||||
* ctx - Compression Context (must be provided)
|
||||
* dctx - Decompression Context (must be provided)
|
||||
* diplayLevel - see BMK_benchFiles
|
||||
* displayName - name used by display
|
||||
* return
|
||||
@@ -138,15 +137,17 @@ BMK_return_t BMK_benchMem(const void* srcBuffer, size_t srcSize,
|
||||
const size_t* fileSizes, unsigned nbFiles,
|
||||
const int cLevel, const ZSTD_compressionParameters* comprParams,
|
||||
const void* dictBuffer, size_t dictBufferSize,
|
||||
ZSTD_CCtx* ctx, ZSTD_DCtx* dctx,
|
||||
int displayLevel, const char* displayName);
|
||||
|
||||
/* See benchMem for normal parameter uses and return, see advancedParams_t for adv */
|
||||
/* See benchMem for normal parameter uses and return, see advancedParams_t for adv
|
||||
* dstBuffer - destination buffer to write compressed output in, NULL if none provided.
|
||||
* dstCapacity - capacity of destination buffer, give 0 if dstBuffer = NULL
|
||||
*/
|
||||
BMK_return_t BMK_benchMemAdvanced(const void* srcBuffer, size_t srcSize,
|
||||
void* dstBuffer, size_t dstCapacity,
|
||||
const size_t* fileSizes, unsigned nbFiles,
|
||||
const int cLevel, const ZSTD_compressionParameters* comprParams,
|
||||
const void* dictBuffer, size_t dictBufferSize,
|
||||
ZSTD_CCtx* ctx, ZSTD_DCtx* dctx,
|
||||
int displayLevel, const char* displayName,
|
||||
const BMK_advancedParams_t* adv);
|
||||
|
||||
@@ -170,8 +171,10 @@ typedef size_t (*BMK_initFn_t)(void*);
|
||||
* srcBuffers - an array of buffers to be operated on by benchFn
|
||||
* srcSizes - an array of the sizes of above buffers
|
||||
* dstBuffers - an array of buffers to be written into by benchFn
|
||||
* dstCapacities - an array of the capacities of above buffers.
|
||||
* dstCapacities - an array of the capacities of above buffers
|
||||
* blockResults - the return value of benchFn called on each block.
|
||||
* nbLoops - defines number of times benchFn is run.
|
||||
* assumed array of size blockCount, will have compressed size of each block written to it.
|
||||
* return
|
||||
* .error will give a nonzero value if ZSTD_isError() is nonzero for any of the return
|
||||
* of the calls to initFn and benchFn, or if benchFunction errors internally
|
||||
@@ -182,12 +185,11 @@ typedef size_t (*BMK_initFn_t)(void*);
|
||||
* into dstBuffer, hence this value will be the total amount of bytes written to
|
||||
* dstBuffer.
|
||||
*/
|
||||
BMK_customReturn_t BMK_benchFunction(
|
||||
BMK_benchFn_t benchFn, void* benchPayload,
|
||||
BMK_customReturn_t BMK_benchFunction(BMK_benchFn_t benchFn, void* benchPayload,
|
||||
BMK_initFn_t initFn, void* initPayload,
|
||||
size_t blockCount,
|
||||
const void* const * const srcBuffers, const size_t* srcSizes,
|
||||
void* const * const dstBuffers, const size_t* dstCapacities,
|
||||
void * const * const dstBuffers, const size_t* dstCapacities, size_t* blockResults,
|
||||
unsigned nbLoops);
|
||||
|
||||
|
||||
@@ -216,7 +218,7 @@ BMK_customTimedReturn_t BMK_benchFunctionTimed(BMK_timedFnState_t* cont,
|
||||
BMK_initFn_t initFn, void* initPayload,
|
||||
size_t blockCount,
|
||||
const void* const * const srcBlockBuffers, const size_t* srcBlockSizes,
|
||||
void* const * const dstBlockBuffers, const size_t* dstBlockCapacities);
|
||||
void* const * const dstBlockBuffers, const size_t* dstBlockCapacities, size_t* blockResults);
|
||||
|
||||
#endif /* BENCH_H_121279284357 */
|
||||
|
||||
|
||||
+2
-2
@@ -129,7 +129,7 @@ zstdmt_d_%.o : $(ZSTDDIR)/decompress/%.c
|
||||
fullbench32: CPPFLAGS += -m32
|
||||
fullbench fullbench32 : CPPFLAGS += $(MULTITHREAD_CPP)
|
||||
fullbench fullbench32 : LDFLAGS += $(MULTITHREAD_LD)
|
||||
fullbench fullbench32 : DEBUGFLAGS = # turn off assert() for speed measurements
|
||||
fullbench fullbench32 : DEBUGFLAGS = -DNDEBUG # turn off assert() for speed measurements
|
||||
fullbench fullbench32 : $(ZSTD_FILES)
|
||||
fullbench fullbench32 : $(PRGDIR)/datagen.c $(PRGDIR)/bench.c fullbench.c
|
||||
$(CC) $(FLAGS) $^ -o $@$(EXT)
|
||||
@@ -200,7 +200,7 @@ zstreamtest-dll : $(ZSTDDIR)/common/xxhash.c # xxh symbols not exposed from dll
|
||||
zstreamtest-dll : $(ZSTREAM_LOCAL_FILES)
|
||||
$(CC) $(CPPFLAGS) $(CFLAGS) $(filter %.c,$^) $(LDFLAGS) -o $@$(EXT)
|
||||
|
||||
paramgrill : DEBUGFLAGS = # turn off assert() for speed measurements
|
||||
paramgrill : DEBUGFLAGS = -DNDEBUG # turn off assert() for speed measurements
|
||||
paramgrill : $(ZSTD_FILES) $(PRGDIR)/bench.c $(PRGDIR)/datagen.c paramgrill.c
|
||||
$(CC) $(FLAGS) $^ -lm -o $@$(EXT)
|
||||
|
||||
|
||||
@@ -88,3 +88,56 @@ as well as the 10,000 original files for more detailed comparison of decompressi
|
||||
will choose a random seed, and for 1 minute,
|
||||
generate random test frames and ensure that the
|
||||
zstd library correctly decompresses them in both simple and streaming modes.
|
||||
|
||||
#### `paramgrill` - tool for generating compression table parameters and optimizing parameters on file given constraints
|
||||
|
||||
Full list of arguments
|
||||
```
|
||||
-T# : set level 1 speed objective
|
||||
-B# : cut input into blocks of size # (default : single block)
|
||||
-S : benchmarks a single run (example command: -Sl3w10h12)
|
||||
w# - windowLog
|
||||
h# - hashLog
|
||||
c# - chainLog
|
||||
s# - searchLog
|
||||
l# - searchLength
|
||||
t# - targetLength
|
||||
S# - strategy
|
||||
L# - level
|
||||
--zstd= : Single run, parameter selection syntax same as zstdcli with more parameters
|
||||
(Added forceAttachDictionary / fadt)
|
||||
When invoked with --optimize, this represents the sample to exceed.
|
||||
--optimize= : find parameters to maximize compression ratio given parameters
|
||||
Can use all --zstd= commands to constrain the type of solution found in addition to the following constraints
|
||||
cSpeed= : Minimum compression speed
|
||||
dSpeed= : Minimum decompression speed
|
||||
cMem= : Maximum compression memory
|
||||
lvl= : Searches for solutions which are strictly better than that compression lvl in ratio and cSpeed,
|
||||
stc= : When invoked with lvl=, represents percentage slack in ratio/cSpeed allowed for a solution to be considered (Default 100%)
|
||||
: In normal operation, represents percentage slack in choosing viable starting strategy selection in choosing the default parameters
|
||||
(Lower value will begin with stronger strategies) (Default 90%)
|
||||
speedRatio= (accepts decimals)
|
||||
: determines value of gains in speed vs gains in ratio
|
||||
when determining overall winner (default 5 (1% ratio = 5% speed)).
|
||||
tries= : Maximum number of random restarts on a single strategy before switching (Default 5)
|
||||
Higher values will make optimizer run longer, more chances to find better solution.
|
||||
memLog : Limits the log of the size of each memotable (1 per strategy). Will use hash tables when state space is larger than max size.
|
||||
Setting memLog = 0 turns off memoization
|
||||
--display= : specifiy which parameters are included in the output
|
||||
can use all --zstd parameter names and 'cParams' as a shorthand for all parameters used in ZSTD_compressionParameters
|
||||
(Default: display all params available)
|
||||
-P# : generated sample compressibility (when no file is provided)
|
||||
-t# : Caps runtime of operation in seconds (default : 99999 seconds (about 27 hours ))
|
||||
-v : Prints Benchmarking output
|
||||
-D : Next argument dictionary file
|
||||
-s : Benchmark all files separately
|
||||
-q : Quiet, repeat for more quiet
|
||||
-q Prints parameters + results whenever a new best is found
|
||||
-qq Only prints parameters whenever a new best is found, prints final parameters + results
|
||||
-qqq Only print final parameters + results
|
||||
-qqqq Only prints final parameter set in the form --zstd=
|
||||
-v : Verbose, cancels quiet, repeat for more volume
|
||||
-v Prints all candidate parameters and results
|
||||
|
||||
```
|
||||
Any inputs afterwards are treated as files to benchmark.
|
||||
|
||||
+3
-2
@@ -336,7 +336,7 @@ size_t local_ZSTD_decompressContinue(const void* src, size_t srcSize, void* dst,
|
||||
static size_t benchMem(const void* src, size_t srcSize, U32 benchNb, int cLevel, ZSTD_compressionParameters* cparams)
|
||||
{
|
||||
BYTE* dstBuff;
|
||||
size_t const dstBuffSize = ZSTD_compressBound(srcSize);
|
||||
size_t dstBuffSize = ZSTD_compressBound(srcSize);
|
||||
void* buff2, *buff1;
|
||||
const char* benchName;
|
||||
BMK_benchFn_t benchFunction;
|
||||
@@ -514,9 +514,10 @@ static size_t benchMem(const void* src, size_t srcSize, U32 benchNb, int cLevel,
|
||||
|
||||
/* benchmark loop */
|
||||
{
|
||||
void* dstBuffv = (void*)dstBuff;
|
||||
r = BMK_benchFunction(benchFunction, buff2,
|
||||
NULL, NULL, 1, &src, &srcSize,
|
||||
(void * const * const)&dstBuff, &dstBuffSize, g_nbIterations);
|
||||
&dstBuffv, &dstBuffSize, NULL, g_nbIterations);
|
||||
if(r.error) {
|
||||
DISPLAY("ERROR %d ! ! \n", r.error);
|
||||
errorcode = r.error;
|
||||
|
||||
+2245
-488
@@ -17,7 +17,6 @@
|
||||
#include <stdio.h> /* fprintf, fopen, ftello64 */
|
||||
#include <string.h> /* strcmp */
|
||||
#include <math.h> /* log */
|
||||
#include <time.h>
|
||||
#include <assert.h>
|
||||
|
||||
#include "mem.h"
|
||||
@@ -27,7 +26,8 @@
|
||||
#include "xxhash.h"
|
||||
#include "util.h"
|
||||
#include "bench.h"
|
||||
|
||||
#include "zstd_errors.h"
|
||||
#include "zstd_internal.h"
|
||||
|
||||
/*-************************************
|
||||
* Constants
|
||||
@@ -36,13 +36,7 @@
|
||||
#define AUTHOR "Yann Collet"
|
||||
#define WELCOME_MESSAGE "*** %s %s %i-bits, by %s ***\n", PROGRAM_DESCRIPTION, ZSTD_VERSION_STRING, (int)(sizeof(void*)*8), AUTHOR
|
||||
|
||||
|
||||
#define KB *(1<<10)
|
||||
#define MB *(1<<20)
|
||||
#define GB *(1ULL<<30)
|
||||
|
||||
#define NBLOOPS 2
|
||||
#define TIMELOOP (2 * SEC_TO_MICRO)
|
||||
#define TIMELOOP_NANOSEC (1*1000000000ULL) /* 1 second */
|
||||
#define NB_LEVELS_TRACKED 22 /* ensured being >= ZSTD_maxCLevel() in BMK_init_level_constraints() */
|
||||
|
||||
static const size_t maxMemory = (sizeof(size_t)==4) ? (2 GB - 64 MB) : (size_t)(1ULL << ((sizeof(size_t)*8)-31));
|
||||
@@ -56,39 +50,302 @@ static const int g_maxNbVariations = 64;
|
||||
* Macros
|
||||
**************************************/
|
||||
#define DISPLAY(...) fprintf(stderr, __VA_ARGS__)
|
||||
#define DISPLAYLEVEL(n, ...) if(g_displayLevel >= n) { fprintf(stderr, __VA_ARGS__); }
|
||||
#define DEBUGOUTPUT(...) { if (DEBUG) DISPLAY(__VA_ARGS__); }
|
||||
|
||||
#define TIMED 0
|
||||
#ifndef DEBUG
|
||||
# define DEBUG 0
|
||||
#endif
|
||||
|
||||
#undef MIN
|
||||
#undef MAX
|
||||
#define MIN(a,b) ( (a) < (b) ? (a) : (b) )
|
||||
#define MAX(a,b) ( (a) > (b) ? (a) : (b) )
|
||||
#define CUSTOM_LEVEL 99
|
||||
#define BASE_CLEVEL 1
|
||||
|
||||
#define FADT_MIN 0
|
||||
#define FADT_MAX ((U32)-1)
|
||||
|
||||
#define ZSTD_TARGETLENGTH_MIN 0
|
||||
#define ZSTD_TARGETLENGTH_MAX 999
|
||||
|
||||
#define WLOG_RANGE (ZSTD_WINDOWLOG_MAX - ZSTD_WINDOWLOG_MIN + 1)
|
||||
#define CLOG_RANGE (ZSTD_CHAINLOG_MAX - ZSTD_CHAINLOG_MIN + 1)
|
||||
#define HLOG_RANGE (ZSTD_HASHLOG_MAX - ZSTD_HASHLOG_MIN + 1)
|
||||
#define SLOG_RANGE (ZSTD_SEARCHLOG_MAX - ZSTD_SEARCHLOG_MIN + 1)
|
||||
#define SLEN_RANGE (ZSTD_SEARCHLENGTH_MAX - ZSTD_SEARCHLENGTH_MIN + 1)
|
||||
#define TLEN_RANGE 17
|
||||
#define STRT_RANGE (ZSTD_btultra - ZSTD_fast + 1)
|
||||
#define FADT_RANGE 3
|
||||
|
||||
#define CHECKTIME(r) { if(BMK_timeSpan(g_time) > g_timeLimit_s) { DEBUGOUTPUT("Time Limit Reached\n"); return r; } }
|
||||
#define CHECKTIMEGT(ret, val, _gototag) {if(BMK_timeSpan(g_time) > g_timeLimit_s) { DEBUGOUTPUT("Time Limit Reached\n"); ret = val; goto _gototag; } }
|
||||
|
||||
#define PARAM_UNSET ((U32)-2) /* can't be -1 b/c fadt uses -1 */
|
||||
|
||||
static const char* g_stratName[ZSTD_btultra+1] = {
|
||||
"(none) ", "ZSTD_fast ", "ZSTD_dfast ",
|
||||
"ZSTD_greedy ", "ZSTD_lazy ", "ZSTD_lazy2 ",
|
||||
"ZSTD_btlazy2 ", "ZSTD_btopt ", "ZSTD_btultra "};
|
||||
|
||||
|
||||
static const U32 tlen_table[TLEN_RANGE] = { 0, 1, 2, 4, 6, 8, 12, 16, 24, 32, 48, 64, 96, 128, 256, 512, 999 };
|
||||
|
||||
/*-************************************
|
||||
* Benchmark Parameters
|
||||
* Setup for Adding new params
|
||||
**************************************/
|
||||
|
||||
static double g_grillDuration_s = 99999; /* about 27 hours */
|
||||
static U32 g_nbIterations = NBLOOPS;
|
||||
static double g_compressibility = COMPRESSIBILITY_DEFAULT;
|
||||
static U32 g_blockSize = 0;
|
||||
static U32 g_rand = 1;
|
||||
static U32 g_singleRun = 0;
|
||||
static U32 g_target = 0;
|
||||
static U32 g_noSeed = 0;
|
||||
static ZSTD_compressionParameters g_params = { 0, 0, 0, 0, 0, 0, ZSTD_greedy };
|
||||
/* indices for each of the variables */
|
||||
typedef enum {
|
||||
wlog_ind = 0,
|
||||
clog_ind = 1,
|
||||
hlog_ind = 2,
|
||||
slog_ind = 3,
|
||||
slen_ind = 4,
|
||||
tlen_ind = 5,
|
||||
strt_ind = 6,
|
||||
fadt_ind = 7, /* forceAttachDict */
|
||||
NUM_PARAMS = 8
|
||||
} varInds_t;
|
||||
|
||||
void BMK_SetNbIterations(int nbLoops)
|
||||
{
|
||||
g_nbIterations = nbLoops;
|
||||
DISPLAY("- %u iterations -\n", g_nbIterations);
|
||||
typedef struct {
|
||||
U32 vals[NUM_PARAMS];
|
||||
} paramValues_t;
|
||||
|
||||
/* maximum value of parameters */
|
||||
static const U32 mintable[NUM_PARAMS] =
|
||||
{ ZSTD_WINDOWLOG_MIN, ZSTD_CHAINLOG_MIN, ZSTD_HASHLOG_MIN, ZSTD_SEARCHLOG_MIN, ZSTD_SEARCHLENGTH_MIN, ZSTD_TARGETLENGTH_MIN, ZSTD_fast, FADT_MIN };
|
||||
|
||||
/* minimum value of parameters */
|
||||
static const U32 maxtable[NUM_PARAMS] =
|
||||
{ ZSTD_WINDOWLOG_MAX, ZSTD_CHAINLOG_MAX, ZSTD_HASHLOG_MAX, ZSTD_SEARCHLOG_MAX, ZSTD_SEARCHLENGTH_MAX, ZSTD_TARGETLENGTH_MAX, ZSTD_btultra, FADT_MAX };
|
||||
|
||||
/* # of values parameters can take on */
|
||||
static const U32 rangetable[NUM_PARAMS] =
|
||||
{ WLOG_RANGE, CLOG_RANGE, HLOG_RANGE, SLOG_RANGE, SLEN_RANGE, TLEN_RANGE, STRT_RANGE, FADT_RANGE };
|
||||
|
||||
/* ZSTD_cctxSetParameter() index to set */
|
||||
static const ZSTD_cParameter cctxSetParamTable[NUM_PARAMS] =
|
||||
{ ZSTD_p_windowLog, ZSTD_p_chainLog, ZSTD_p_hashLog, ZSTD_p_searchLog, ZSTD_p_minMatch, ZSTD_p_targetLength, ZSTD_p_compressionStrategy, ZSTD_p_forceAttachDict };
|
||||
|
||||
/* names of parameters */
|
||||
static const char* g_paramNames[NUM_PARAMS] =
|
||||
{ "windowLog", "chainLog", "hashLog","searchLog", "searchLength", "targetLength", "strategy", "forceAttachDict" };
|
||||
|
||||
/* shortened names of parameters */
|
||||
static const char* g_shortParamNames[NUM_PARAMS] =
|
||||
{ "wlog", "clog", "hlog","slog", "slen", "tlen", "strt", "fadt" };
|
||||
|
||||
/* maps value from { 0 to rangetable[param] - 1 } to valid paramvalues */
|
||||
static U32 rangeMap(varInds_t param, int ind) {
|
||||
ind = MAX(MIN(ind, (int)rangetable[param] - 1), 0);
|
||||
switch(param) {
|
||||
case tlen_ind:
|
||||
return tlen_table[ind];
|
||||
case fadt_ind: /* 0, 1, 2 -> -1, 0, 1 */
|
||||
return ind - 1;
|
||||
case wlog_ind: /* using default: triggers -Wswitch-enum */
|
||||
case clog_ind:
|
||||
case hlog_ind:
|
||||
case slog_ind:
|
||||
case slen_ind:
|
||||
case strt_ind:
|
||||
return mintable[param] + ind;
|
||||
case NUM_PARAMS:
|
||||
DISPLAY("Error, not a valid param\n ");
|
||||
return (U32)-1;
|
||||
}
|
||||
return 0; /* should never happen, stop compiler warnings */
|
||||
}
|
||||
|
||||
/* inverse of rangeMap */
|
||||
static int invRangeMap(varInds_t param, U32 value) {
|
||||
value = MIN(MAX(mintable[param], value), maxtable[param]);
|
||||
switch(param) {
|
||||
case tlen_ind: /* bin search */
|
||||
{
|
||||
int lo = 0;
|
||||
int hi = TLEN_RANGE;
|
||||
while(lo < hi) {
|
||||
int mid = (lo + hi) / 2;
|
||||
if(tlen_table[mid] < value) {
|
||||
lo = mid + 1;
|
||||
} if(tlen_table[mid] == value) {
|
||||
return mid;
|
||||
} else {
|
||||
hi = mid;
|
||||
}
|
||||
}
|
||||
return lo;
|
||||
}
|
||||
case fadt_ind:
|
||||
return (int)value + 1;
|
||||
case wlog_ind:
|
||||
case clog_ind:
|
||||
case hlog_ind:
|
||||
case slog_ind:
|
||||
case slen_ind:
|
||||
case strt_ind:
|
||||
return value - mintable[param];
|
||||
case NUM_PARAMS:
|
||||
DISPLAY("Error, not a valid param\n ");
|
||||
return -2;
|
||||
}
|
||||
return 0; /* should never happen, stop compiler warnings */
|
||||
}
|
||||
|
||||
/* display of params */
|
||||
static void displayParamVal(FILE* f, varInds_t param, U32 value, int width) {
|
||||
switch(param) {
|
||||
case fadt_ind: if(width) { fprintf(f, "%*d", width, (int)value); } else { fprintf(f, "%d", (int)value); } break;
|
||||
case strt_ind: if(width) { fprintf(f, "%*s", width, g_stratName[value]); } else { fprintf(f, "%s", g_stratName[value]); } break;
|
||||
case wlog_ind:
|
||||
case clog_ind:
|
||||
case hlog_ind:
|
||||
case slog_ind:
|
||||
case slen_ind:
|
||||
case tlen_ind: if(width) { fprintf(f, "%*u", width, value); } else { fprintf(f, "%u", value); } break;
|
||||
case NUM_PARAMS:
|
||||
DISPLAY("Error, not a valid param\n "); break;
|
||||
}
|
||||
}
|
||||
|
||||
/*-************************************
|
||||
* Benchmark Parameters/Global Variables
|
||||
**************************************/
|
||||
|
||||
typedef BYTE U8;
|
||||
|
||||
/* General Utility */
|
||||
static U32 g_timeLimit_s = 99999; /* about 27 hours */
|
||||
static UTIL_time_t g_time; /* to be used to compare solution finding speeds to compare to original */
|
||||
static U32 g_blockSize = 0;
|
||||
static U32 g_rand = 1;
|
||||
|
||||
/* Display */
|
||||
static int g_displayLevel = 3;
|
||||
static BYTE g_silenceParams[NUM_PARAMS];
|
||||
|
||||
/* Mode Selection */
|
||||
static U32 g_singleRun = 0;
|
||||
static U32 g_optimizer = 0;
|
||||
static int g_optmode = 0;
|
||||
|
||||
/* For cLevel Table generation */
|
||||
static U32 g_target = 0;
|
||||
static U32 g_noSeed = 0;
|
||||
|
||||
/* For optimizer */
|
||||
static paramValues_t g_params; /* Initialized at the beginning of main w/ emptyParams() function */
|
||||
static double g_ratioMultiplier = 5.;
|
||||
static U32 g_strictness = PARAM_UNSET; /* range 1 - 100, measure of how strict */
|
||||
static BMK_result_t g_lvltarget;
|
||||
|
||||
typedef enum {
|
||||
directMap,
|
||||
xxhashMap,
|
||||
noMemo
|
||||
} memoTableType_t;
|
||||
|
||||
typedef struct {
|
||||
memoTableType_t tableType;
|
||||
BYTE* table;
|
||||
size_t tableLen;
|
||||
varInds_t varArray[NUM_PARAMS];
|
||||
size_t varLen;
|
||||
} memoTable_t;
|
||||
|
||||
typedef struct {
|
||||
BMK_result_t result;
|
||||
paramValues_t params;
|
||||
} winnerInfo_t;
|
||||
|
||||
typedef struct {
|
||||
U32 cSpeed; /* bytes / sec */
|
||||
U32 dSpeed;
|
||||
U32 cMem; /* bytes */
|
||||
} constraint_t;
|
||||
|
||||
typedef struct winner_ll_node winner_ll_node;
|
||||
struct winner_ll_node {
|
||||
winnerInfo_t res;
|
||||
winner_ll_node* next;
|
||||
};
|
||||
|
||||
static winner_ll_node* g_winners; /* linked list sorted ascending by cSize & cSpeed */
|
||||
|
||||
/*
|
||||
* Additional Global Variables (Defined Above Use)
|
||||
* g_level_constraint
|
||||
* g_alreadyTested
|
||||
* g_maxTries
|
||||
* g_clockGranularity
|
||||
*/
|
||||
|
||||
/*-*******************************************************
|
||||
* Private functions
|
||||
* General Util Functions
|
||||
*********************************************************/
|
||||
|
||||
/* accuracy in seconds only, span can be multiple years */
|
||||
static double BMK_timeSpan(time_t tStart) { return difftime(time(NULL), tStart); }
|
||||
/* nullified useless params, to ensure count stats */
|
||||
/* cleans up params for memoizing / display */
|
||||
static paramValues_t sanitizeParams(paramValues_t params)
|
||||
{
|
||||
if (params.vals[strt_ind] == ZSTD_fast)
|
||||
params.vals[clog_ind] = 0, params.vals[slog_ind] = 0;
|
||||
if (params.vals[strt_ind] == ZSTD_dfast)
|
||||
params.vals[slog_ind] = 0;
|
||||
if (params.vals[strt_ind] != ZSTD_btopt && params.vals[strt_ind] != ZSTD_btultra && params.vals[strt_ind] != ZSTD_fast)
|
||||
params.vals[tlen_ind] = 0;
|
||||
|
||||
return params;
|
||||
}
|
||||
|
||||
static ZSTD_compressionParameters pvalsToCParams(paramValues_t p) {
|
||||
ZSTD_compressionParameters c;
|
||||
memset(&c, 0, sizeof(ZSTD_compressionParameters));
|
||||
c.windowLog = p.vals[wlog_ind];
|
||||
c.chainLog = p.vals[clog_ind];
|
||||
c.hashLog = p.vals[hlog_ind];
|
||||
c.searchLog = p.vals[slog_ind];
|
||||
c.searchLength = p.vals[slen_ind];
|
||||
c.targetLength = p.vals[tlen_ind];
|
||||
c.strategy = p.vals[strt_ind];
|
||||
/* no forceAttachDict */
|
||||
return c;
|
||||
}
|
||||
|
||||
static paramValues_t cParamsToPVals(ZSTD_compressionParameters c) {
|
||||
paramValues_t p;
|
||||
varInds_t i;
|
||||
p.vals[wlog_ind] = c.windowLog;
|
||||
p.vals[clog_ind] = c.chainLog;
|
||||
p.vals[hlog_ind] = c.hashLog;
|
||||
p.vals[slog_ind] = c.searchLog;
|
||||
p.vals[slen_ind] = c.searchLength;
|
||||
p.vals[tlen_ind] = c.targetLength;
|
||||
p.vals[strt_ind] = c.strategy;
|
||||
|
||||
/* set all other params to their minimum value */
|
||||
for(i = strt_ind + 1; i < NUM_PARAMS; i++) {
|
||||
p.vals[i] = mintable[i];
|
||||
}
|
||||
return p;
|
||||
}
|
||||
|
||||
/* equivalent of ZSTD_adjustCParams for paramValues_t */
|
||||
static paramValues_t adjustParams(paramValues_t p, const size_t maxBlockSize, const size_t dictSize) {
|
||||
paramValues_t ot = p;
|
||||
varInds_t i;
|
||||
p = cParamsToPVals(ZSTD_adjustCParams(pvalsToCParams(p), maxBlockSize, dictSize));
|
||||
if(!dictSize) { p.vals[fadt_ind] = 0; }
|
||||
/* retain value of all other parameters */
|
||||
for(i = strt_ind + 1; i < NUM_PARAMS; i++) {
|
||||
p.vals[i] = ot.vals[i];
|
||||
}
|
||||
return p;
|
||||
}
|
||||
|
||||
static size_t BMK_findMaxMem(U64 requiredMem)
|
||||
{
|
||||
@@ -98,16 +355,18 @@ static size_t BMK_findMaxMem(U64 requiredMem)
|
||||
requiredMem = (((requiredMem >> 26) + 1) << 26);
|
||||
if (requiredMem > maxMemory) requiredMem = maxMemory;
|
||||
|
||||
requiredMem += 2*step;
|
||||
while (!testmem) {
|
||||
requiredMem -= step;
|
||||
requiredMem += 2 * step;
|
||||
while (!testmem && requiredMem > 0) {
|
||||
testmem = malloc ((size_t)requiredMem);
|
||||
requiredMem -= step;
|
||||
}
|
||||
|
||||
free (testmem);
|
||||
return (size_t) (requiredMem - step);
|
||||
return (size_t) requiredMem;
|
||||
}
|
||||
|
||||
/* accuracy in seconds only, span can be multiple years */
|
||||
static U32 BMK_timeSpan(const UTIL_time_t tStart) { return (U32)(UTIL_clockSpanMicro(tStart) / 1000000ULL); }
|
||||
|
||||
static U32 FUZ_rotl32(U32 x, U32 r)
|
||||
{
|
||||
@@ -126,75 +385,467 @@ U32 FUZ_rand(U32* src)
|
||||
return rand32 >> 5;
|
||||
}
|
||||
|
||||
/** longCommandWArg() :
|
||||
* check if *stringPtr is the same as longCommand.
|
||||
* If yes, @return 1 and advances *stringPtr to the position which immediately follows longCommand.
|
||||
* @return 0 and doesn't modify *stringPtr otherwise.
|
||||
* from zstdcli.c
|
||||
/* allows zeros */
|
||||
#define CLAMPCHECK(val,min,max) { \
|
||||
if (((val)<(min)) | ((val)>(max))) { \
|
||||
DISPLAY("INVALID PARAMETER CONSTRAINTS\n"); \
|
||||
return 0; \
|
||||
} }
|
||||
|
||||
static int paramValid(const paramValues_t paramTarget) {
|
||||
U32 i;
|
||||
for(i = 0; i < NUM_PARAMS; i++) {
|
||||
CLAMPCHECK(paramTarget.vals[i], mintable[i], maxtable[i]);
|
||||
}
|
||||
return 1;
|
||||
}
|
||||
|
||||
static paramValues_t cParamUnsetMin(paramValues_t paramTarget) {
|
||||
varInds_t i;
|
||||
for(i = 0; i < NUM_PARAMS; i++) {
|
||||
if(paramTarget.vals[i] == PARAM_UNSET) {
|
||||
paramTarget.vals[i] = mintable[i];
|
||||
}
|
||||
}
|
||||
return paramTarget;
|
||||
}
|
||||
|
||||
static paramValues_t emptyParams(void) {
|
||||
U32 i;
|
||||
paramValues_t p;
|
||||
for(i = 0; i < NUM_PARAMS; i++) {
|
||||
p.vals[i] = PARAM_UNSET;
|
||||
}
|
||||
return p;
|
||||
}
|
||||
|
||||
static winnerInfo_t initWinnerInfo(const paramValues_t p) {
|
||||
winnerInfo_t w1;
|
||||
w1.result.cSpeed = 0.;
|
||||
w1.result.dSpeed = 0.;
|
||||
w1.result.cMem = (size_t)-1;
|
||||
w1.result.cSize = (size_t)-1;
|
||||
w1.params = p;
|
||||
return w1;
|
||||
}
|
||||
|
||||
static paramValues_t overwriteParams(paramValues_t base, const paramValues_t mask) {
|
||||
U32 i;
|
||||
for(i = 0; i < NUM_PARAMS; i++) {
|
||||
if(mask.vals[i] != PARAM_UNSET) {
|
||||
base.vals[i] = mask.vals[i];
|
||||
}
|
||||
}
|
||||
return base;
|
||||
}
|
||||
|
||||
static void paramVaryOnce(const varInds_t paramIndex, const int amt, paramValues_t* ptr) {
|
||||
ptr->vals[paramIndex] = rangeMap(paramIndex, invRangeMap(paramIndex, ptr->vals[paramIndex]) + amt);
|
||||
}
|
||||
|
||||
/* varies ptr by nbChanges respecting varyParams*/
|
||||
static void paramVariation(paramValues_t* ptr, memoTable_t* mtAll, const U32 nbChanges)
|
||||
{
|
||||
paramValues_t p;
|
||||
U32 validated = 0;
|
||||
while (!validated) {
|
||||
U32 i;
|
||||
p = *ptr;
|
||||
for (i = 0 ; i < nbChanges ; i++) {
|
||||
const U32 changeID = (U32)FUZ_rand(&g_rand) % (mtAll[p.vals[strt_ind]].varLen << 1);
|
||||
paramVaryOnce(mtAll[p.vals[strt_ind]].varArray[changeID >> 1], ((changeID & 1) << 1) - 1, &p);
|
||||
}
|
||||
validated = paramValid(p);
|
||||
}
|
||||
*ptr = p;
|
||||
}
|
||||
|
||||
/* Completely random parameter selection */
|
||||
static paramValues_t randomParams(void)
|
||||
{
|
||||
varInds_t v; paramValues_t p;
|
||||
for(v = 0; v <= NUM_PARAMS; v++) {
|
||||
p.vals[v] = rangeMap(v, FUZ_rand(&g_rand) % rangetable[v]);
|
||||
}
|
||||
return p;
|
||||
}
|
||||
|
||||
static U64 g_clockGranularity = 100000000ULL;
|
||||
|
||||
static void findClockGranularity(void) {
|
||||
UTIL_time_t clockStart = UTIL_getTime();
|
||||
U64 el1 = 0, el2 = 0;
|
||||
int i = 0;
|
||||
do {
|
||||
el1 = el2;
|
||||
el2 = UTIL_clockSpanNano(clockStart);
|
||||
if(el1 < el2) {
|
||||
U64 iv = el2 - el1;
|
||||
if(g_clockGranularity > iv) {
|
||||
g_clockGranularity = iv;
|
||||
i = 0;
|
||||
} else {
|
||||
i++;
|
||||
}
|
||||
}
|
||||
} while(i < 10);
|
||||
DEBUGOUTPUT("Granularity: %llu\n", (unsigned long long)g_clockGranularity);
|
||||
}
|
||||
|
||||
/*-************************************
|
||||
* Optimizer Util Functions
|
||||
**************************************/
|
||||
|
||||
/* checks results are feasible */
|
||||
static int feasible(const BMK_result_t results, const constraint_t target) {
|
||||
return (results.cSpeed >= target.cSpeed) && (results.dSpeed >= target.dSpeed) && (results.cMem <= target.cMem) && (!g_optmode || results.cSize <= g_lvltarget.cSize);
|
||||
}
|
||||
|
||||
/* hill climbing value for part 1 */
|
||||
/* Scoring here is a linear reward for all set constraints normalized between 0 to 1
|
||||
* (with 0 at 0 and 1 being fully fulfilling the constraint), summed with a logarithmic
|
||||
* bonus to exceeding the constraint value. We also give linear ratio for compression ratio.
|
||||
* The constant factors are experimental.
|
||||
*/
|
||||
static unsigned longCommandWArg(const char** stringPtr, const char* longCommand)
|
||||
{
|
||||
size_t const comSize = strlen(longCommand);
|
||||
int const result = !strncmp(*stringPtr, longCommand, comSize);
|
||||
if (result) *stringPtr += comSize;
|
||||
return result;
|
||||
static double resultScore(const BMK_result_t res, const size_t srcSize, const constraint_t target) {
|
||||
double cs = 0., ds = 0., rt, cm = 0.;
|
||||
const double r1 = 1, r2 = 0.1, rtr = 0.5;
|
||||
double ret;
|
||||
if(target.cSpeed) { cs = res.cSpeed / (double)target.cSpeed; }
|
||||
if(target.dSpeed) { ds = res.dSpeed / (double)target.dSpeed; }
|
||||
if(target.cMem != (U32)-1) { cm = (double)target.cMem / res.cMem; }
|
||||
rt = ((double)srcSize / res.cSize);
|
||||
|
||||
ret = (MIN(1, cs) + MIN(1, ds) + MIN(1, cm))*r1 + rt * rtr +
|
||||
(MAX(0, log(cs))+ MAX(0, log(ds))+ MAX(0, log(cm))) * r2;
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
/*-*******************************************************
|
||||
* Bench functions
|
||||
*********************************************************/
|
||||
|
||||
typedef struct
|
||||
{
|
||||
const char* srcPtr;
|
||||
size_t srcSize;
|
||||
char* cPtr;
|
||||
size_t cRoom;
|
||||
size_t cSize;
|
||||
char* resPtr;
|
||||
size_t resSize;
|
||||
} blockParam_t;
|
||||
|
||||
|
||||
const char* g_stratName[ZSTD_btultra+1] = {
|
||||
"(none) ", "ZSTD_fast ", "ZSTD_dfast ",
|
||||
"ZSTD_greedy ", "ZSTD_lazy ", "ZSTD_lazy2 ",
|
||||
"ZSTD_btlazy2 ", "ZSTD_btopt ", "ZSTD_btultra "};
|
||||
|
||||
/* TODO: support additional parameters (more files, fileSizes) */
|
||||
static size_t
|
||||
BMK_benchParam(BMK_result_t* resultPtr,
|
||||
const void* srcBuffer, size_t srcSize,
|
||||
ZSTD_CCtx* ctx, ZSTD_DCtx* dctx,
|
||||
const ZSTD_compressionParameters cParams) {
|
||||
|
||||
BMK_return_t res = BMK_benchMem(srcBuffer,srcSize, &srcSize, 1, 0, &cParams, NULL, 0, ctx, dctx, 0, "File");
|
||||
*resultPtr = res.result;
|
||||
return res.error;
|
||||
/* calculates normalized squared euclidean distance of result1 if it is in the first quadrant relative to lvlRes */
|
||||
static double resultDistLvl(const BMK_result_t result1, const BMK_result_t lvlRes) {
|
||||
double normalizedCSpeedGain1 = (result1.cSpeed / lvlRes.cSpeed) - 1;
|
||||
double normalizedRatioGain1 = ((double)lvlRes.cSize / result1.cSize) - 1;
|
||||
if(normalizedRatioGain1 < 0 || normalizedCSpeedGain1 < 0) {
|
||||
return 0.0;
|
||||
}
|
||||
return normalizedRatioGain1 * g_ratioMultiplier + normalizedCSpeedGain1;
|
||||
}
|
||||
|
||||
static void BMK_printWinner(FILE* f, U32 cLevel, BMK_result_t result, ZSTD_compressionParameters params, size_t srcSize)
|
||||
/* return true if r2 strictly better than r1 */
|
||||
static int compareResultLT(const BMK_result_t result1, const BMK_result_t result2, const constraint_t target, size_t srcSize) {
|
||||
if(feasible(result1, target) && feasible(result2, target)) {
|
||||
if(g_optmode) {
|
||||
return resultDistLvl(result1, g_lvltarget) < resultDistLvl(result2, g_lvltarget);
|
||||
} else {
|
||||
return (result1.cSize > result2.cSize) || (result1.cSize == result2.cSize && result2.cSpeed > result1.cSpeed)
|
||||
|| (result1.cSize == result2.cSize && result2.cSpeed == result1.cSpeed && result2.dSpeed > result1.dSpeed);
|
||||
}
|
||||
}
|
||||
return feasible(result2, target) || (!feasible(result1, target) && (resultScore(result1, srcSize, target) < resultScore(result2, srcSize, target)));
|
||||
}
|
||||
|
||||
static constraint_t relaxTarget(constraint_t target) {
|
||||
target.cMem = (U32)-1;
|
||||
target.cSpeed *= ((double)g_strictness) / 100;
|
||||
target.dSpeed *= ((double)g_strictness) / 100;
|
||||
return target;
|
||||
}
|
||||
|
||||
static void optimizerAdjustInput(paramValues_t* pc, const size_t maxBlockSize) {
|
||||
varInds_t v;
|
||||
for(v = 0; v < NUM_PARAMS; v++) {
|
||||
if(pc->vals[v] != PARAM_UNSET) {
|
||||
U32 newval = MIN(MAX(pc->vals[v], mintable[v]), maxtable[v]);
|
||||
if(newval != pc->vals[v]) {
|
||||
pc->vals[v] = newval;
|
||||
DISPLAY("Warning: parameter %s not in valid range, adjusting to ", g_paramNames[v]); displayParamVal(stderr, v, newval, 0); DISPLAY("\n");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if(pc->vals[wlog_ind] != PARAM_UNSET) {
|
||||
|
||||
U32 sshb = maxBlockSize > 1 ? ZSTD_highbit32((U32)(maxBlockSize-1)) + 1 : 1;
|
||||
/* edge case of highBit not working for 0 */
|
||||
|
||||
if(maxBlockSize < (1ULL << 31) && sshb + 1 < pc->vals[wlog_ind]) {
|
||||
U32 adjust = MAX(mintable[wlog_ind], sshb);
|
||||
if(adjust != pc->vals[wlog_ind]) {
|
||||
pc->vals[wlog_ind] = adjust;
|
||||
DISPLAY("Warning: windowLog larger than src/block size, adjusted to %u\n", pc->vals[wlog_ind]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if(pc->vals[wlog_ind] != PARAM_UNSET && pc->vals[clog_ind] != PARAM_UNSET) {
|
||||
U32 maxclog;
|
||||
if(pc->vals[strt_ind] == PARAM_UNSET || pc->vals[strt_ind] >= (U32)ZSTD_btlazy2) {
|
||||
maxclog = pc->vals[wlog_ind] + 1;
|
||||
} else {
|
||||
maxclog = pc->vals[wlog_ind];
|
||||
}
|
||||
|
||||
if(pc->vals[clog_ind] > maxclog) {
|
||||
pc->vals[clog_ind] = maxclog;
|
||||
DISPLAY("Warning: chainlog too much larger than windowLog size, adjusted to %u\n", pc->vals[clog_ind]);
|
||||
}
|
||||
}
|
||||
|
||||
if(pc->vals[wlog_ind] != PARAM_UNSET && pc->vals[hlog_ind] != PARAM_UNSET) {
|
||||
if(pc->vals[wlog_ind] + 1 < pc->vals[hlog_ind]) {
|
||||
pc->vals[hlog_ind] = pc->vals[wlog_ind] + 1;
|
||||
DISPLAY("Warning: hashlog too much larger than windowLog size, adjusted to %u\n", pc->vals[hlog_ind]);
|
||||
}
|
||||
}
|
||||
|
||||
if(pc->vals[slog_ind] != PARAM_UNSET && pc->vals[clog_ind] != PARAM_UNSET) {
|
||||
if(pc->vals[slog_ind] > pc->vals[clog_ind]) {
|
||||
pc->vals[clog_ind] = pc->vals[slog_ind];
|
||||
DISPLAY("Warning: searchLog larger than chainLog, adjusted to %u\n", pc->vals[slog_ind]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static int redundantParams(const paramValues_t paramValues, const constraint_t target, const size_t maxBlockSize) {
|
||||
return
|
||||
(ZSTD_estimateCStreamSize_usingCParams(pvalsToCParams(paramValues)) > (size_t)target.cMem) /* Uses too much memory */
|
||||
|| ((1ULL << (paramValues.vals[wlog_ind] - 1)) >= maxBlockSize && paramValues.vals[wlog_ind] != mintable[wlog_ind]) /* wlog too much bigger than src size */
|
||||
|| (paramValues.vals[clog_ind] > (paramValues.vals[wlog_ind] + (paramValues.vals[strt_ind] > ZSTD_btlazy2))) /* chainLog larger than windowLog*/
|
||||
|| (paramValues.vals[slog_ind] > paramValues.vals[clog_ind]) /* searchLog larger than chainLog */
|
||||
|| (paramValues.vals[hlog_ind] > paramValues.vals[wlog_ind] + 1); /* hashLog larger than windowLog + 1 */
|
||||
|
||||
}
|
||||
|
||||
/*-************************************
|
||||
* Display Functions
|
||||
**************************************/
|
||||
|
||||
static void BMK_translateAdvancedParams(FILE* f, const paramValues_t params) {
|
||||
varInds_t v;
|
||||
int first = 1;
|
||||
fprintf(f,"--zstd=");
|
||||
for(v = 0; v < NUM_PARAMS; v++) {
|
||||
if(g_silenceParams[v]) { continue; }
|
||||
if(!first) { fprintf(f, ","); }
|
||||
fprintf(f,"%s=", g_paramNames[v]);
|
||||
|
||||
if(v == strt_ind) { fprintf(f,"%u", params.vals[v]); }
|
||||
else { displayParamVal(f, v, params.vals[v], 0); }
|
||||
first = 0;
|
||||
}
|
||||
fprintf(f, "\n");
|
||||
}
|
||||
|
||||
static void BMK_displayOneResult(FILE* f, winnerInfo_t res, const size_t srcSize) {
|
||||
varInds_t v;
|
||||
int first = 1;
|
||||
res.params = cParamUnsetMin(res.params);
|
||||
fprintf(f," {");
|
||||
for(v = 0; v < NUM_PARAMS; v++) {
|
||||
if(g_silenceParams[v]) { continue; }
|
||||
if(!first) { fprintf(f, ","); }
|
||||
displayParamVal(f, v, res.params.vals[v], 3);
|
||||
first = 0;
|
||||
}
|
||||
|
||||
fprintf(f, " }, /* R:%5.3f at %5.1f MB/s - %5.1f MB/s */\n",
|
||||
(double)srcSize / res.result.cSize, (double)res.result.cSpeed / (1 MB), (double)res.result.dSpeed / (1 MB));
|
||||
}
|
||||
|
||||
/* Writes to f the results of a parameter benchmark */
|
||||
/* when used with --optimize, will only print results better than previously discovered */
|
||||
static void BMK_printWinner(FILE* f, const int cLevel, const BMK_result_t result, const paramValues_t params, const size_t srcSize)
|
||||
{
|
||||
char lvlstr[15] = "Custom Level";
|
||||
DISPLAY("\r%79s\r", "");
|
||||
fprintf(f," {%3u,%3u,%3u,%3u,%3u,%3u, %s }, ",
|
||||
params.windowLog, params.chainLog, params.hashLog, params.searchLog, params.searchLength,
|
||||
params.targetLength, g_stratName[(U32)(params.strategy)]);
|
||||
winnerInfo_t w;
|
||||
w.params = params;
|
||||
w.result = result;
|
||||
|
||||
fprintf(f, "\r%79s\r", "");
|
||||
|
||||
if(cLevel != CUSTOM_LEVEL) {
|
||||
snprintf(lvlstr, 15, " Level %2u ", cLevel);
|
||||
snprintf(lvlstr, 15, " Level %2d ", cLevel);
|
||||
}
|
||||
fprintf(f,
|
||||
"/* %s */ /* R:%5.3f at %5.1f MB/s - %5.1f MB/s */\n",
|
||||
lvlstr, (double)srcSize / result.cSize, result.cSpeed / 1000000., result.dSpeed / 1000000.);
|
||||
|
||||
if(TIMED) {
|
||||
const U64 time = UTIL_clockSpanNano(g_time);
|
||||
const U64 minutes = time / (60ULL * TIMELOOP_NANOSEC);
|
||||
fprintf(f, "%1lu:%2lu:%05.2f - ", (unsigned long) minutes / 60,(unsigned long) minutes % 60, (double)(time - minutes * TIMELOOP_NANOSEC * 60ULL)/TIMELOOP_NANOSEC);
|
||||
}
|
||||
|
||||
fprintf(f, "/* %s */ ", lvlstr);
|
||||
BMK_displayOneResult(f, w, srcSize);
|
||||
}
|
||||
|
||||
/* comparison function: */
|
||||
/* strictly better, strictly worse, equal, speed-side adv, size-side adv */
|
||||
#define WORSE_RESULT 0
|
||||
#define BETTER_RESULT 1
|
||||
#define ERROR_RESULT 2
|
||||
|
||||
typedef struct {
|
||||
BMK_result_t result;
|
||||
ZSTD_compressionParameters params;
|
||||
} winnerInfo_t;
|
||||
#define SPEED_RESULT 4
|
||||
#define SIZE_RESULT 5
|
||||
/* maybe have epsilon-eq to limit table size? */
|
||||
static int speedSizeCompare(const BMK_result_t r1, const BMK_result_t r2) {
|
||||
if(r1.cSpeed < r2.cSpeed) {
|
||||
if(r1.cSize >= r2.cSize) {
|
||||
return BETTER_RESULT;
|
||||
}
|
||||
return SPEED_RESULT; /* r2 is smaller but not faster. */
|
||||
} else {
|
||||
if(r1.cSize <= r2.cSize) {
|
||||
return WORSE_RESULT;
|
||||
}
|
||||
return SIZE_RESULT; /* r2 is faster but not smaller */
|
||||
}
|
||||
}
|
||||
|
||||
static void BMK_printWinners2(FILE* f, const winnerInfo_t* winners, size_t srcSize)
|
||||
/* 0 for insertion, 1 for no insert */
|
||||
/* maintain invariant speedSizeCompare(n, n->next) = SPEED_RESULT */
|
||||
static int insertWinner(const winnerInfo_t w, const constraint_t targetConstraints) {
|
||||
BMK_result_t r = w.result;
|
||||
winner_ll_node* cur_node = g_winners;
|
||||
/* first node to insert */
|
||||
if(!feasible(r, targetConstraints)) {
|
||||
return 1;
|
||||
}
|
||||
|
||||
if(g_winners == NULL) {
|
||||
winner_ll_node* first_node = malloc(sizeof(winner_ll_node));
|
||||
if(first_node == NULL) {
|
||||
return 1;
|
||||
}
|
||||
first_node->next = NULL;
|
||||
first_node->res = w;
|
||||
g_winners = first_node;
|
||||
return 0;
|
||||
}
|
||||
|
||||
while(cur_node->next != NULL) {
|
||||
switch(speedSizeCompare(cur_node->res.result, r)) {
|
||||
case WORSE_RESULT:
|
||||
{
|
||||
return 1; /* never insert if better */
|
||||
}
|
||||
case BETTER_RESULT:
|
||||
{
|
||||
winner_ll_node* tmp;
|
||||
cur_node->res = cur_node->next->res;
|
||||
tmp = cur_node->next;
|
||||
cur_node->next = cur_node->next->next;
|
||||
free(tmp);
|
||||
break;
|
||||
}
|
||||
case SIZE_RESULT:
|
||||
{
|
||||
cur_node = cur_node->next;
|
||||
break;
|
||||
}
|
||||
case SPEED_RESULT: /* insert after first size result, then return */
|
||||
{
|
||||
winner_ll_node* newnode = malloc(sizeof(winner_ll_node));
|
||||
if(newnode == NULL) {
|
||||
return 1;
|
||||
}
|
||||
newnode->res = cur_node->res;
|
||||
cur_node->res = w;
|
||||
newnode->next = cur_node->next;
|
||||
cur_node->next = newnode;
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
assert(cur_node->next == NULL);
|
||||
switch(speedSizeCompare(cur_node->res.result, r)) {
|
||||
case WORSE_RESULT:
|
||||
{
|
||||
return 1; /* never insert if better */
|
||||
}
|
||||
case BETTER_RESULT:
|
||||
{
|
||||
cur_node->res = w;
|
||||
return 0;
|
||||
}
|
||||
case SIZE_RESULT:
|
||||
{
|
||||
winner_ll_node* newnode = malloc(sizeof(winner_ll_node));
|
||||
if(newnode == NULL) {
|
||||
return 1;
|
||||
}
|
||||
newnode->res = w;
|
||||
newnode->next = NULL;
|
||||
cur_node->next = newnode;
|
||||
return 0;
|
||||
}
|
||||
case SPEED_RESULT: /* insert before first size result, then return */
|
||||
{
|
||||
winner_ll_node* newnode = malloc(sizeof(winner_ll_node));
|
||||
if(newnode == NULL) {
|
||||
return 1;
|
||||
}
|
||||
newnode->res = cur_node->res;
|
||||
cur_node->res = w;
|
||||
newnode->next = cur_node->next;
|
||||
cur_node->next = newnode;
|
||||
return 0;
|
||||
}
|
||||
default:
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
|
||||
static void BMK_printWinnerOpt(FILE* f, const U32 cLevel, const BMK_result_t result, const paramValues_t params, const constraint_t targetConstraints, const size_t srcSize)
|
||||
{
|
||||
/* global winner used for constraints */
|
||||
/* cSize, cSpeed, dSpeed, cMem */
|
||||
static winnerInfo_t g_winner = { { (size_t)-1LL, 0, 0, (size_t)-1LL }, { { PARAM_UNSET, PARAM_UNSET, PARAM_UNSET, PARAM_UNSET, PARAM_UNSET, PARAM_UNSET, PARAM_UNSET, PARAM_UNSET } } };
|
||||
if(DEBUG || compareResultLT(g_winner.result, result, targetConstraints, srcSize) || g_displayLevel >= 4) {
|
||||
if(DEBUG && compareResultLT(g_winner.result, result, targetConstraints, srcSize)) {
|
||||
DISPLAY("New Winner: \n");
|
||||
}
|
||||
|
||||
if(g_displayLevel >= 2) { BMK_printWinner(f, cLevel, result, params, srcSize); }
|
||||
|
||||
if(compareResultLT(g_winner.result, result, targetConstraints, srcSize)) {
|
||||
if(g_displayLevel >= 1) { BMK_translateAdvancedParams(f, params); }
|
||||
g_winner.result = result;
|
||||
g_winner.params = params;
|
||||
}
|
||||
}
|
||||
|
||||
if(g_optmode && g_optimizer && (DEBUG || g_displayLevel == 3)) {
|
||||
winnerInfo_t w;
|
||||
winner_ll_node* n;
|
||||
w.result = result;
|
||||
w.params = params;
|
||||
insertWinner(w, targetConstraints);
|
||||
|
||||
if(!DEBUG) { fprintf(f, "\033c"); }
|
||||
fprintf(f, "\n");
|
||||
|
||||
/* the table */
|
||||
fprintf(f, "================================\n");
|
||||
for(n = g_winners; n != NULL; n = n->next) {
|
||||
BMK_displayOneResult(f, n->res, srcSize);
|
||||
}
|
||||
fprintf(f, "================================\n");
|
||||
fprintf(f, "Level Bounds: R: > %.3f AND C: < %.1f MB/s \n\n",
|
||||
(double)srcSize / g_lvltarget.cSize, (double)g_lvltarget.cSpeed / (1 MB));
|
||||
|
||||
|
||||
fprintf(f, "Overall Winner: \n");
|
||||
BMK_displayOneResult(f, g_winner, srcSize);
|
||||
BMK_translateAdvancedParams(f, g_winner.params);
|
||||
|
||||
fprintf(f, "Latest BMK: \n");\
|
||||
BMK_displayOneResult(f, w, srcSize);
|
||||
}
|
||||
}
|
||||
|
||||
static void BMK_printWinners2(FILE* f, const winnerInfo_t* winners, const size_t srcSize)
|
||||
{
|
||||
int cLevel;
|
||||
|
||||
@@ -206,7 +857,7 @@ static void BMK_printWinners2(FILE* f, const winnerInfo_t* winners, size_t srcSi
|
||||
}
|
||||
|
||||
|
||||
static void BMK_printWinners(FILE* f, const winnerInfo_t* winners, size_t srcSize)
|
||||
static void BMK_printWinners(FILE* f, const winnerInfo_t* winners, const size_t srcSize)
|
||||
{
|
||||
fseek(f, 0, SEEK_SET);
|
||||
BMK_printWinners2(f, winners, srcSize);
|
||||
@@ -215,9 +866,774 @@ static void BMK_printWinners(FILE* f, const winnerInfo_t* winners, size_t srcSiz
|
||||
}
|
||||
|
||||
|
||||
/*-*******************************************************
|
||||
* Functions to Benchmark
|
||||
*********************************************************/
|
||||
|
||||
typedef struct {
|
||||
double cSpeed_min;
|
||||
double dSpeed_min;
|
||||
ZSTD_CCtx* ctx;
|
||||
const void* dictBuffer;
|
||||
size_t dictBufferSize;
|
||||
int cLevel;
|
||||
const paramValues_t* comprParams;
|
||||
} BMK_initCCtxArgs;
|
||||
|
||||
static size_t local_initCCtx(void* payload) {
|
||||
const BMK_initCCtxArgs* ag = (const BMK_initCCtxArgs*)payload;
|
||||
varInds_t i;
|
||||
ZSTD_CCtx_reset(ag->ctx);
|
||||
ZSTD_CCtx_resetParameters(ag->ctx);
|
||||
ZSTD_CCtx_setParameter(ag->ctx, ZSTD_p_compressionLevel, ag->cLevel);
|
||||
|
||||
for(i = 0; i < NUM_PARAMS; i++) {
|
||||
if(ag->comprParams->vals[i] != PARAM_UNSET)
|
||||
ZSTD_CCtx_setParameter(ag->ctx, cctxSetParamTable[i], ag->comprParams->vals[i]);
|
||||
}
|
||||
ZSTD_CCtx_loadDictionary(ag->ctx, ag->dictBuffer, ag->dictBufferSize);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
typedef struct {
|
||||
ZSTD_DCtx* dctx;
|
||||
const void* dictBuffer;
|
||||
size_t dictBufferSize;
|
||||
} BMK_initDCtxArgs;
|
||||
|
||||
static size_t local_initDCtx(void* payload) {
|
||||
const BMK_initDCtxArgs* ag = (const BMK_initDCtxArgs*)payload;
|
||||
ZSTD_DCtx_reset(ag->dctx);
|
||||
ZSTD_DCtx_loadDictionary(ag->dctx, ag->dictBuffer, ag->dictBufferSize);
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* additional argument is just the context */
|
||||
static size_t local_defaultCompress(
|
||||
const void* srcBuffer, size_t srcSize,
|
||||
void* dstBuffer, size_t dstSize,
|
||||
void* addArgs) {
|
||||
size_t moreToFlush = 1;
|
||||
ZSTD_CCtx* ctx = (ZSTD_CCtx*)addArgs;
|
||||
ZSTD_inBuffer in;
|
||||
ZSTD_outBuffer out;
|
||||
in.src = srcBuffer;
|
||||
in.size = srcSize;
|
||||
in.pos = 0;
|
||||
out.dst = dstBuffer;
|
||||
out.size = dstSize;
|
||||
out.pos = 0;
|
||||
assert(dstSize == ZSTD_compressBound(srcSize)); /* specific to this version, which is only used in paramgrill */
|
||||
while (moreToFlush) {
|
||||
if(out.pos == out.size) {
|
||||
return (size_t)-ZSTD_error_dstSize_tooSmall;
|
||||
}
|
||||
moreToFlush = ZSTD_compress_generic(ctx, &out, &in, ZSTD_e_end);
|
||||
if (ZSTD_isError(moreToFlush)) {
|
||||
return moreToFlush;
|
||||
}
|
||||
}
|
||||
return out.pos;
|
||||
}
|
||||
|
||||
/* additional argument is just the context */
|
||||
static size_t local_defaultDecompress(
|
||||
const void* srcBuffer, size_t srcSize,
|
||||
void* dstBuffer, size_t dstSize,
|
||||
void* addArgs) {
|
||||
size_t moreToFlush = 1;
|
||||
ZSTD_DCtx* dctx = (ZSTD_DCtx*)addArgs;
|
||||
ZSTD_inBuffer in;
|
||||
ZSTD_outBuffer out;
|
||||
in.src = srcBuffer;
|
||||
in.size = srcSize;
|
||||
in.pos = 0;
|
||||
out.dst = dstBuffer;
|
||||
out.size = dstSize;
|
||||
out.pos = 0;
|
||||
while (moreToFlush) {
|
||||
if(out.pos == out.size) {
|
||||
return (size_t)-ZSTD_error_dstSize_tooSmall;
|
||||
}
|
||||
moreToFlush = ZSTD_decompress_generic(dctx,
|
||||
&out, &in);
|
||||
if (ZSTD_isError(moreToFlush)) {
|
||||
return moreToFlush;
|
||||
}
|
||||
}
|
||||
return out.pos;
|
||||
|
||||
}
|
||||
|
||||
/*-************************************
|
||||
* Data Initialization Functions
|
||||
**************************************/
|
||||
|
||||
typedef struct {
|
||||
void* srcBuffer;
|
||||
size_t srcSize;
|
||||
const void** srcPtrs;
|
||||
size_t* srcSizes;
|
||||
void** dstPtrs;
|
||||
size_t* dstCapacities;
|
||||
size_t* dstSizes;
|
||||
void** resPtrs;
|
||||
size_t* resSizes;
|
||||
size_t nbBlocks;
|
||||
size_t maxBlockSize;
|
||||
} buffers_t;
|
||||
|
||||
typedef struct {
|
||||
size_t dictSize;
|
||||
void* dictBuffer;
|
||||
ZSTD_CCtx* cctx;
|
||||
ZSTD_DCtx* dctx;
|
||||
} contexts_t;
|
||||
|
||||
static void freeNonSrcBuffers(const buffers_t b) {
|
||||
free(b.srcPtrs);
|
||||
free(b.srcSizes);
|
||||
|
||||
if(b.dstPtrs != NULL) {
|
||||
free(b.dstPtrs[0]);
|
||||
}
|
||||
free(b.dstPtrs);
|
||||
free(b.dstCapacities);
|
||||
free(b.dstSizes);
|
||||
|
||||
if(b.resPtrs != NULL) {
|
||||
free(b.resPtrs[0]);
|
||||
}
|
||||
free(b.resPtrs);
|
||||
free(b.resSizes);
|
||||
}
|
||||
|
||||
static void freeBuffers(const buffers_t b) {
|
||||
if(b.srcPtrs != NULL) {
|
||||
free(b.srcBuffer);
|
||||
}
|
||||
freeNonSrcBuffers(b);
|
||||
}
|
||||
|
||||
/* srcBuffer will be freed by freeBuffers now */
|
||||
static int createBuffersFromMemory(buffers_t* buff, void * srcBuffer, const size_t nbFiles,
|
||||
const size_t* fileSizes)
|
||||
{
|
||||
size_t pos = 0, n, blockSize;
|
||||
U32 maxNbBlocks, blockNb = 0;
|
||||
buff->srcSize = 0;
|
||||
for(n = 0; n < nbFiles; n++) {
|
||||
buff->srcSize += fileSizes[n];
|
||||
}
|
||||
|
||||
if(buff->srcSize == 0) {
|
||||
DISPLAY("No data to bench\n");
|
||||
return 1;
|
||||
}
|
||||
|
||||
blockSize = g_blockSize ? g_blockSize : buff->srcSize;
|
||||
maxNbBlocks = (U32) ((buff->srcSize + (blockSize-1)) / blockSize) + (U32)nbFiles;
|
||||
|
||||
buff->srcPtrs = (const void**)calloc(maxNbBlocks, sizeof(void*));
|
||||
buff->srcSizes = (size_t*)malloc(maxNbBlocks * sizeof(size_t));
|
||||
|
||||
buff->dstPtrs = (void**)calloc(maxNbBlocks, sizeof(void*));
|
||||
buff->dstCapacities = (size_t*)malloc(maxNbBlocks * sizeof(size_t));
|
||||
buff->dstSizes = (size_t*)malloc(maxNbBlocks * sizeof(size_t));
|
||||
|
||||
buff->resPtrs = (void**)calloc(maxNbBlocks, sizeof(void*));
|
||||
buff->resSizes = (size_t*)malloc(maxNbBlocks * sizeof(size_t));
|
||||
|
||||
if(!buff->srcPtrs || !buff->srcSizes || !buff->dstPtrs || !buff->dstCapacities || !buff->dstSizes || !buff->resPtrs || !buff->resSizes) {
|
||||
DISPLAY("alloc error\n");
|
||||
freeNonSrcBuffers(*buff);
|
||||
return 1;
|
||||
}
|
||||
|
||||
buff->srcBuffer = srcBuffer;
|
||||
buff->srcPtrs[0] = (const void*)buff->srcBuffer;
|
||||
buff->dstPtrs[0] = malloc(ZSTD_compressBound(buff->srcSize) + (maxNbBlocks * 1024));
|
||||
buff->resPtrs[0] = malloc(buff->srcSize);
|
||||
|
||||
if(!buff->dstPtrs[0] || !buff->resPtrs[0]) {
|
||||
DISPLAY("alloc error\n");
|
||||
freeNonSrcBuffers(*buff);
|
||||
return 1;
|
||||
}
|
||||
|
||||
for(n = 0; n < nbFiles; n++) {
|
||||
size_t pos_end = pos + fileSizes[n];
|
||||
for(; pos < pos_end; blockNb++) {
|
||||
buff->srcPtrs[blockNb] = (const void*)((char*)srcBuffer + pos);
|
||||
buff->srcSizes[blockNb] = blockSize;
|
||||
pos += blockSize;
|
||||
}
|
||||
|
||||
if(fileSizes[n] > 0) { buff->srcSizes[blockNb - 1] = ((fileSizes[n] - 1) % blockSize) + 1; }
|
||||
pos = pos_end;
|
||||
}
|
||||
|
||||
buff->dstCapacities[0] = ZSTD_compressBound(buff->srcSizes[0]);
|
||||
buff->dstSizes[0] = buff->dstCapacities[0];
|
||||
buff->resSizes[0] = buff->srcSizes[0];
|
||||
buff->maxBlockSize = buff->srcSizes[0];
|
||||
|
||||
for(n = 1; n < blockNb; n++) {
|
||||
buff->dstPtrs[n] = ((char*)buff->dstPtrs[n-1]) + buff->dstCapacities[n-1];
|
||||
buff->resPtrs[n] = ((char*)buff->resPtrs[n-1]) + buff->resSizes[n-1];
|
||||
buff->dstCapacities[n] = ZSTD_compressBound(buff->srcSizes[n]);
|
||||
buff->dstSizes[n] = buff->dstCapacities[n];
|
||||
buff->resSizes[n] = buff->srcSizes[n];
|
||||
|
||||
buff->maxBlockSize = MAX(buff->maxBlockSize, buff->srcSizes[n]);
|
||||
}
|
||||
|
||||
buff->nbBlocks = blockNb;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* allocates buffer's arguments. returns success / failuere */
|
||||
static int createBuffers(buffers_t* buff, const char* const * const fileNamesTable,
|
||||
size_t nbFiles) {
|
||||
size_t pos = 0;
|
||||
size_t n;
|
||||
size_t totalSizeToLoad = UTIL_getTotalFileSize(fileNamesTable, (U32)nbFiles);
|
||||
size_t benchedSize = MIN(BMK_findMaxMem(totalSizeToLoad * 3) / 3, totalSizeToLoad);
|
||||
size_t* fileSizes = calloc(sizeof(size_t), nbFiles);
|
||||
void* srcBuffer = NULL;
|
||||
int ret = 0;
|
||||
|
||||
if(!totalSizeToLoad || !benchedSize) {
|
||||
ret = 1;
|
||||
DISPLAY("Nothing to Bench\n");
|
||||
goto _cleanUp;
|
||||
}
|
||||
|
||||
srcBuffer = malloc(benchedSize);
|
||||
|
||||
if(!fileSizes || !srcBuffer) {
|
||||
ret = 1;
|
||||
goto _cleanUp;
|
||||
}
|
||||
|
||||
for(n = 0; n < nbFiles; n++) {
|
||||
FILE* f;
|
||||
U64 fileSize = UTIL_getFileSize(fileNamesTable[n]);
|
||||
if (UTIL_isDirectory(fileNamesTable[n])) {
|
||||
DISPLAY("Ignoring %s directory... \n", fileNamesTable[n]);
|
||||
continue;
|
||||
}
|
||||
if (fileSize == UTIL_FILESIZE_UNKNOWN) {
|
||||
DISPLAY("Cannot evaluate size of %s, ignoring ... \n", fileNamesTable[n]);
|
||||
continue;
|
||||
}
|
||||
f = fopen(fileNamesTable[n], "rb");
|
||||
if (f==NULL) {
|
||||
DISPLAY("impossible to open file %s\n", fileNamesTable[n]);
|
||||
fclose(f);
|
||||
ret = 10;
|
||||
goto _cleanUp;
|
||||
}
|
||||
|
||||
DISPLAYLEVEL(2, "Loading %s... \r", fileNamesTable[n]);
|
||||
|
||||
if (fileSize + pos > benchedSize) fileSize = benchedSize - pos, nbFiles=n; /* buffer too small - stop after this file */
|
||||
{
|
||||
char* buffer = (char*)(srcBuffer);
|
||||
size_t const readSize = fread((buffer)+pos, 1, (size_t)fileSize, f);
|
||||
fclose(f);
|
||||
if (readSize != (size_t)fileSize) {
|
||||
DISPLAY("could not read %s", fileNamesTable[n]);
|
||||
ret = 1;
|
||||
goto _cleanUp;
|
||||
}
|
||||
|
||||
fileSizes[n] = readSize;
|
||||
pos += readSize;
|
||||
}
|
||||
}
|
||||
|
||||
ret = createBuffersFromMemory(buff, srcBuffer, nbFiles, fileSizes);
|
||||
|
||||
_cleanUp:
|
||||
if(ret) { free(srcBuffer); }
|
||||
free(fileSizes);
|
||||
return ret;
|
||||
}
|
||||
|
||||
static void freeContexts(const contexts_t ctx) {
|
||||
free(ctx.dictBuffer);
|
||||
ZSTD_freeCCtx(ctx.cctx);
|
||||
ZSTD_freeDCtx(ctx.dctx);
|
||||
}
|
||||
|
||||
static int createContexts(contexts_t* ctx, const char* dictFileName) {
|
||||
FILE* f;
|
||||
size_t readSize;
|
||||
ctx->cctx = ZSTD_createCCtx();
|
||||
ctx->dctx = ZSTD_createDCtx();
|
||||
if(dictFileName == NULL) {
|
||||
ctx->dictSize = 0;
|
||||
ctx->dictBuffer = NULL;
|
||||
return 0;
|
||||
}
|
||||
ctx->dictSize = UTIL_getFileSize(dictFileName);
|
||||
ctx->dictBuffer = malloc(ctx->dictSize);
|
||||
|
||||
f = fopen(dictFileName, "rb");
|
||||
|
||||
if(!f) {
|
||||
DISPLAY("unable to open file\n");
|
||||
fclose(f);
|
||||
freeContexts(*ctx);
|
||||
return 1;
|
||||
}
|
||||
|
||||
if(ctx->dictSize > 64 MB || !(ctx->dictBuffer)) {
|
||||
DISPLAY("dictionary too large\n");
|
||||
fclose(f);
|
||||
freeContexts(*ctx);
|
||||
return 1;
|
||||
}
|
||||
readSize = fread(ctx->dictBuffer, 1, ctx->dictSize, f);
|
||||
if(readSize != ctx->dictSize) {
|
||||
DISPLAY("unable to read file\n");
|
||||
fclose(f);
|
||||
freeContexts(*ctx);
|
||||
return 1;
|
||||
}
|
||||
fclose(f);
|
||||
return 0;
|
||||
}
|
||||
|
||||
/*-************************************
|
||||
* Optimizer Memoization Functions
|
||||
**************************************/
|
||||
|
||||
/* return: new length */
|
||||
/* keep old array, will need if iter over strategy. */
|
||||
/* prunes useless params */
|
||||
static size_t sanitizeVarArray(varInds_t* varNew, const size_t varLength, const varInds_t* varArray, const ZSTD_strategy strat) {
|
||||
size_t i, j = 0;
|
||||
for(i = 0; i < varLength; i++) {
|
||||
if( !((varArray[i] == clog_ind && strat == ZSTD_fast)
|
||||
|| (varArray[i] == slog_ind && strat == ZSTD_fast)
|
||||
|| (varArray[i] == slog_ind && strat == ZSTD_dfast)
|
||||
|| (varArray[i] == tlen_ind && strat != ZSTD_btopt && strat != ZSTD_btultra && strat != ZSTD_fast))) {
|
||||
varNew[j] = varArray[i];
|
||||
j++;
|
||||
}
|
||||
}
|
||||
return j;
|
||||
}
|
||||
|
||||
/* res should be NUM_PARAMS size */
|
||||
/* constructs varArray from paramValues_t style parameter */
|
||||
/* pass in using dict. */
|
||||
static size_t variableParams(const paramValues_t paramConstraints, varInds_t* res, const int usingDictionary) {
|
||||
varInds_t i;
|
||||
size_t j = 0;
|
||||
for(i = 0; i < NUM_PARAMS; i++) {
|
||||
if(paramConstraints.vals[i] == PARAM_UNSET) {
|
||||
if(i == fadt_ind && !usingDictionary) continue; /* don't use fadt if no dictionary */
|
||||
res[j] = i; j++;
|
||||
}
|
||||
}
|
||||
return j;
|
||||
}
|
||||
|
||||
/* length of memo table given free variables */
|
||||
static size_t memoTableLen(const varInds_t* varyParams, const size_t varyLen) {
|
||||
size_t arrayLen = 1;
|
||||
size_t i;
|
||||
for(i = 0; i < varyLen; i++) {
|
||||
if(varyParams[i] == strt_ind) continue; /* strategy separated by table */
|
||||
arrayLen *= rangetable[varyParams[i]];
|
||||
}
|
||||
return arrayLen;
|
||||
}
|
||||
|
||||
/* returns unique index in memotable of compression parameters */
|
||||
static unsigned memoTableIndDirect(const paramValues_t* ptr, const varInds_t* varyParams, const size_t varyLen) {
|
||||
size_t i;
|
||||
unsigned ind = 0;
|
||||
for(i = 0; i < varyLen; i++) {
|
||||
varInds_t v = varyParams[i];
|
||||
if(v == strt_ind) continue; /* exclude strategy from memotable */
|
||||
ind *= rangetable[v]; ind += (unsigned)invRangeMap(v, ptr->vals[v]);
|
||||
}
|
||||
return ind;
|
||||
}
|
||||
|
||||
static size_t memoTableGet(const memoTable_t* memoTableArray, const paramValues_t p) {
|
||||
const memoTable_t mt = memoTableArray[p.vals[strt_ind]];
|
||||
switch(mt.tableType) {
|
||||
case directMap:
|
||||
return mt.table[memoTableIndDirect(&p, mt.varArray, mt.varLen)];
|
||||
case xxhashMap:
|
||||
return mt.table[(XXH64(&p.vals, sizeof(U32) * NUM_PARAMS, 0) >> 3) % mt.tableLen];
|
||||
case noMemo:
|
||||
return 0;
|
||||
}
|
||||
return 0; /* should never happen, stop compiler warnings */
|
||||
}
|
||||
|
||||
static void memoTableSet(const memoTable_t* memoTableArray, const paramValues_t p, const BYTE value) {
|
||||
const memoTable_t mt = memoTableArray[p.vals[strt_ind]];
|
||||
switch(mt.tableType) {
|
||||
case directMap:
|
||||
mt.table[memoTableIndDirect(&p, mt.varArray, mt.varLen)] = value; break;
|
||||
case xxhashMap:
|
||||
mt.table[(XXH64(&p.vals, sizeof(U32) * NUM_PARAMS, 0) >> 3) % mt.tableLen] = value; break;
|
||||
case noMemo:
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
/* frees all allocated memotables */
|
||||
static void freeMemoTableArray(memoTable_t* const mtAll) {
|
||||
int i;
|
||||
if(mtAll == NULL) { return; }
|
||||
for(i = 1; i <= (int)ZSTD_btultra; i++) {
|
||||
free(mtAll[i].table);
|
||||
}
|
||||
free(mtAll);
|
||||
}
|
||||
|
||||
/* inits memotables for all (including mallocs), all strategies */
|
||||
/* takes unsanitized varyParams */
|
||||
static memoTable_t* createMemoTableArray(const paramValues_t p, const varInds_t* const varyParams, const size_t varyLen, const U32 memoTableLog) {
|
||||
memoTable_t* mtAll = (memoTable_t*)calloc(sizeof(memoTable_t),(ZSTD_btultra + 1));
|
||||
ZSTD_strategy i, stratMin = ZSTD_fast, stratMax = ZSTD_btultra;
|
||||
|
||||
if(mtAll == NULL) {
|
||||
return NULL;
|
||||
}
|
||||
|
||||
for(i = 1; i <= (int)ZSTD_btultra; i++) {
|
||||
mtAll[i].varLen = sanitizeVarArray(mtAll[i].varArray, varyLen, varyParams, i);
|
||||
}
|
||||
|
||||
/* no memoization */
|
||||
if(memoTableLog == 0) {
|
||||
for(i = 1; i <= (int)ZSTD_btultra; i++) {
|
||||
mtAll[i].tableType = noMemo;
|
||||
mtAll[i].table = NULL;
|
||||
mtAll[i].tableLen = 0;
|
||||
}
|
||||
return mtAll;
|
||||
}
|
||||
|
||||
|
||||
if(p.vals[strt_ind] != PARAM_UNSET) {
|
||||
stratMin = p.vals[strt_ind];
|
||||
stratMax = p.vals[strt_ind];
|
||||
}
|
||||
|
||||
|
||||
for(i = stratMin; i <= stratMax; i++) {
|
||||
size_t mtl = memoTableLen(mtAll[i].varArray, mtAll[i].varLen);
|
||||
mtAll[i].tableType = directMap;
|
||||
|
||||
if(memoTableLog != PARAM_UNSET && mtl > (1ULL << memoTableLog)) { /* use hash table */ /* provide some option to only use hash tables? */
|
||||
mtAll[i].tableType = xxhashMap;
|
||||
mtl = (1ULL << memoTableLog);
|
||||
}
|
||||
|
||||
mtAll[i].table = (BYTE*)calloc(sizeof(BYTE), mtl);
|
||||
mtAll[i].tableLen = mtl;
|
||||
|
||||
if(mtAll[i].table == NULL) {
|
||||
freeMemoTableArray(mtAll);
|
||||
return NULL;
|
||||
}
|
||||
}
|
||||
|
||||
return mtAll;
|
||||
}
|
||||
|
||||
/* Sets pc to random unmeasured set of parameters */
|
||||
/* specifiy strategy */
|
||||
static void randomConstrainedParams(paramValues_t* pc, const memoTable_t* memoTableArray, const ZSTD_strategy st)
|
||||
{
|
||||
size_t j;
|
||||
const memoTable_t mt = memoTableArray[st];
|
||||
pc->vals[strt_ind] = st;
|
||||
for(j = 0; j < mt.tableLen; j++) {
|
||||
int i;
|
||||
for(i = 0; i < NUM_PARAMS; i++) {
|
||||
varInds_t v = mt.varArray[i];
|
||||
if(v == strt_ind) continue;
|
||||
pc->vals[v] = rangeMap(v, FUZ_rand(&g_rand) % rangetable[v]);
|
||||
}
|
||||
|
||||
if(!(memoTableGet(memoTableArray, *pc))) break; /* only pick unpicked params. */
|
||||
}
|
||||
}
|
||||
|
||||
/*-************************************
|
||||
* Benchmarking Functions
|
||||
**************************************/
|
||||
|
||||
/* Replicate functionality of benchMemAdvanced, but with pre-split src / dst buffers */
|
||||
/* The purpose is so that sufficient information is returned so that a decompression call to benchMemInvertible is possible */
|
||||
/* BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, fileSizes, nbFiles, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); */
|
||||
/* nbSeconds used in same way as in BMK_advancedParams_t, as nbIters when in iterMode */
|
||||
|
||||
/* if in decodeOnly, then srcPtr's will be compressed blocks, and uncompressedBlocks will be written to dstPtrs */
|
||||
/* dictionary nullable, nothing else though. */
|
||||
static BMK_return_t BMK_benchMemInvertible(const buffers_t buf, const contexts_t ctx,
|
||||
const int cLevel, const paramValues_t* comprParams,
|
||||
const BMK_mode_t mode, const BMK_loopMode_t loopMode, const unsigned nbSeconds) {
|
||||
|
||||
U32 i;
|
||||
BMK_return_t results = { { 0, 0., 0., 0 }, 0 } ;
|
||||
const void *const *const srcPtrs = (const void *const *const)buf.srcPtrs;
|
||||
size_t const *const srcSizes = buf.srcSizes;
|
||||
void** const dstPtrs = buf.dstPtrs;
|
||||
size_t const *const dstCapacities = buf.dstCapacities;
|
||||
size_t* const dstSizes = buf.dstSizes;
|
||||
void** const resPtrs = buf.resPtrs;
|
||||
size_t const *const resSizes = buf.resSizes;
|
||||
const void* dictBuffer = ctx.dictBuffer;
|
||||
const size_t dictBufferSize = ctx.dictSize;
|
||||
const size_t nbBlocks = buf.nbBlocks;
|
||||
const size_t srcSize = buf.srcSize;
|
||||
ZSTD_CCtx* cctx = ctx.cctx;
|
||||
ZSTD_DCtx* dctx = ctx.dctx;
|
||||
|
||||
/* warmimg up memory */
|
||||
for(i = 0; i < buf.nbBlocks; i++) {
|
||||
if(mode != BMK_decodeOnly) {
|
||||
RDG_genBuffer(dstPtrs[i], dstCapacities[i], 0.10, 0.50, 1);
|
||||
} else {
|
||||
RDG_genBuffer(resPtrs[i], resSizes[i], 0.10, 0.50, 1);
|
||||
}
|
||||
}
|
||||
|
||||
/* Bench */
|
||||
{
|
||||
/* init args */
|
||||
BMK_initCCtxArgs cctxprep;
|
||||
BMK_initDCtxArgs dctxprep;
|
||||
cctxprep.ctx = cctx;
|
||||
cctxprep.dictBuffer = dictBuffer;
|
||||
cctxprep.dictBufferSize = dictBufferSize;
|
||||
cctxprep.cLevel = cLevel;
|
||||
cctxprep.comprParams = comprParams;
|
||||
dctxprep.dctx = dctx;
|
||||
dctxprep.dictBuffer = dictBuffer;
|
||||
dctxprep.dictBufferSize = dictBufferSize;
|
||||
|
||||
if(loopMode == BMK_timeMode) {
|
||||
BMK_customTimedReturn_t intermediateResultCompress;
|
||||
BMK_customTimedReturn_t intermediateResultDecompress;
|
||||
BMK_timedFnState_t* timeStateCompress = BMK_createTimeState(nbSeconds);
|
||||
BMK_timedFnState_t* timeStateDecompress = BMK_createTimeState(nbSeconds);
|
||||
if(mode == BMK_compressOnly) {
|
||||
intermediateResultCompress.completed = 0;
|
||||
intermediateResultDecompress.completed = 1;
|
||||
} else if (mode == BMK_decodeOnly) {
|
||||
intermediateResultCompress.completed = 1;
|
||||
intermediateResultDecompress.completed = 0;
|
||||
} else { /* both */
|
||||
intermediateResultCompress.completed = 0;
|
||||
intermediateResultDecompress.completed = 0;
|
||||
}
|
||||
|
||||
while(!intermediateResultCompress.completed) {
|
||||
intermediateResultCompress = BMK_benchFunctionTimed(timeStateCompress, &local_defaultCompress, (void*)cctx, &local_initCCtx, (void*)&cctxprep,
|
||||
nbBlocks, srcPtrs, srcSizes, dstPtrs, dstCapacities, dstSizes);
|
||||
|
||||
if(intermediateResultCompress.result.error) {
|
||||
results.error = intermediateResultCompress.result.error;
|
||||
BMK_freeTimeState(timeStateCompress);
|
||||
BMK_freeTimeState(timeStateDecompress);
|
||||
return results;
|
||||
}
|
||||
results.result.cSpeed = (srcSize * TIMELOOP_NANOSEC) / intermediateResultCompress.result.result.nanoSecPerRun;
|
||||
results.result.cSize = intermediateResultCompress.result.result.sumOfReturn;
|
||||
}
|
||||
|
||||
while(!intermediateResultDecompress.completed) {
|
||||
intermediateResultDecompress = BMK_benchFunctionTimed(timeStateDecompress, &local_defaultDecompress, (void*)(dctx), &local_initDCtx, (void*)&dctxprep,
|
||||
nbBlocks, (const void* const*)dstPtrs, dstSizes, resPtrs, resSizes, NULL);
|
||||
|
||||
if(intermediateResultDecompress.result.error) {
|
||||
results.error = intermediateResultDecompress.result.error;
|
||||
BMK_freeTimeState(timeStateCompress);
|
||||
BMK_freeTimeState(timeStateDecompress);
|
||||
return results;
|
||||
}
|
||||
results.result.dSpeed = (srcSize * TIMELOOP_NANOSEC) / intermediateResultDecompress.result.result.nanoSecPerRun;
|
||||
}
|
||||
|
||||
BMK_freeTimeState(timeStateCompress);
|
||||
BMK_freeTimeState(timeStateDecompress);
|
||||
|
||||
} else { /* iterMode; */
|
||||
if(mode != BMK_decodeOnly) {
|
||||
|
||||
BMK_customReturn_t compressionResults = BMK_benchFunction(&local_defaultCompress, (void*)cctx, &local_initCCtx, (void*)&cctxprep,
|
||||
nbBlocks, srcPtrs, srcSizes, dstPtrs, dstCapacities, dstSizes, nbSeconds);
|
||||
if(compressionResults.error) {
|
||||
results.error = compressionResults.error;
|
||||
return results;
|
||||
}
|
||||
if(compressionResults.result.nanoSecPerRun == 0) {
|
||||
results.result.cSpeed = 0;
|
||||
} else {
|
||||
results.result.cSpeed = srcSize * TIMELOOP_NANOSEC / compressionResults.result.nanoSecPerRun;
|
||||
}
|
||||
results.result.cSize = compressionResults.result.sumOfReturn;
|
||||
}
|
||||
|
||||
if(mode != BMK_compressOnly) {
|
||||
BMK_customReturn_t decompressionResults;
|
||||
decompressionResults = BMK_benchFunction(
|
||||
&local_defaultDecompress, (void*)(dctx),
|
||||
&local_initDCtx, (void*)&dctxprep, nbBlocks,
|
||||
(const void* const*)dstPtrs, dstSizes, resPtrs, resSizes, NULL,
|
||||
nbSeconds);
|
||||
|
||||
if(decompressionResults.error) {
|
||||
results.error = decompressionResults.error;
|
||||
return results;
|
||||
}
|
||||
|
||||
if(decompressionResults.result.nanoSecPerRun == 0) {
|
||||
results.result.dSpeed = 0;
|
||||
} else {
|
||||
results.result.dSpeed = srcSize * TIMELOOP_NANOSEC / decompressionResults.result.nanoSecPerRun;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
/* Bench */
|
||||
results.result.cMem = (1 << (comprParams->vals[wlog_ind])) + ZSTD_sizeof_CCtx(cctx);
|
||||
return results;
|
||||
}
|
||||
|
||||
static int BMK_benchParam(BMK_result_t* resultPtr,
|
||||
const buffers_t buf, const contexts_t ctx,
|
||||
const paramValues_t cParams) {
|
||||
BMK_return_t res = BMK_benchMemInvertible(buf, ctx, BASE_CLEVEL, &cParams, BMK_both, BMK_timeMode, 3);
|
||||
*resultPtr = res.result;
|
||||
return res.error;
|
||||
}
|
||||
|
||||
|
||||
#define CBENCHMARK(conditional, resultvar, tmpret, mode, loopmode, sec) { \
|
||||
if(conditional) { \
|
||||
BMK_return_t tmpret = BMK_benchMemInvertible(buf, ctx, BASE_CLEVEL, &cParams, mode, loopmode, sec); \
|
||||
if(tmpret.error) { DEBUGOUTPUT("Benchmarking failed\n"); return ERROR_RESULT; } \
|
||||
if(mode != BMK_decodeOnly) { \
|
||||
resultvar.cSpeed = tmpret.result.cSpeed; \
|
||||
resultvar.cSize = tmpret.result.cSize; \
|
||||
resultvar.cMem = tmpret.result.cMem; \
|
||||
} \
|
||||
if(mode != BMK_compressOnly) { resultvar.dSpeed = tmpret.result.dSpeed; } \
|
||||
} \
|
||||
}
|
||||
|
||||
/* Benchmarking which stops when we are sufficiently sure the solution is infeasible / worse than the winner */
|
||||
#define VARIANCE 1.2
|
||||
static int allBench(BMK_result_t* resultPtr,
|
||||
const buffers_t buf, const contexts_t ctx,
|
||||
const paramValues_t cParams,
|
||||
const constraint_t target,
|
||||
BMK_result_t* winnerResult, int feas) {
|
||||
BMK_result_t resultMax, benchres;
|
||||
U64 loopDurationC = 0, loopDurationD = 0;
|
||||
double uncertaintyConstantC = 3., uncertaintyConstantD = 3.;
|
||||
double winnerRS;
|
||||
/* initial benchmarking, gives exact ratio and memory, warms up future runs */
|
||||
CBENCHMARK(1, benchres, tmp, BMK_both, BMK_iterMode, 1);
|
||||
|
||||
winnerRS = resultScore(*winnerResult, buf.srcSize, target);
|
||||
DEBUGOUTPUT("WinnerScore: %f\n ", winnerRS);
|
||||
|
||||
*resultPtr = benchres;
|
||||
|
||||
/* calculate uncertainty in compression / decompression runs */
|
||||
if(benchres.cSpeed) {
|
||||
loopDurationC = ((buf.srcSize * TIMELOOP_NANOSEC) / benchres.cSpeed);
|
||||
uncertaintyConstantC = ((loopDurationC + (double)(2 * g_clockGranularity))/loopDurationC);
|
||||
}
|
||||
|
||||
if(benchres.dSpeed) {
|
||||
loopDurationD = ((buf.srcSize * TIMELOOP_NANOSEC) / benchres.dSpeed);
|
||||
uncertaintyConstantD = ((loopDurationD + (double)(2 * g_clockGranularity))/loopDurationD);
|
||||
}
|
||||
|
||||
/* anything with worse ratio in feas is definitely worse, discard */
|
||||
if(feas && benchres.cSize < winnerResult->cSize && !g_optmode) {
|
||||
return WORSE_RESULT;
|
||||
}
|
||||
|
||||
/* second run, if first run is too short, gives approximate cSpeed + dSpeed */
|
||||
CBENCHMARK(loopDurationC < TIMELOOP_NANOSEC / 10, benchres, tmp, BMK_compressOnly, BMK_iterMode, 1);
|
||||
CBENCHMARK(loopDurationD < TIMELOOP_NANOSEC / 10, benchres, tmp, BMK_decodeOnly, BMK_iterMode, 1);
|
||||
|
||||
*resultPtr = benchres;
|
||||
|
||||
/* optimistic assumption of benchres */
|
||||
resultMax = benchres;
|
||||
resultMax.cSpeed *= uncertaintyConstantC * VARIANCE;
|
||||
resultMax.dSpeed *= uncertaintyConstantD * VARIANCE;
|
||||
|
||||
/* disregard infeasible results in feas mode */
|
||||
/* disregard if resultMax < winner in infeas mode */
|
||||
if((feas && !feasible(resultMax, target)) ||
|
||||
(!feas && (winnerRS > resultScore(resultMax, buf.srcSize, target)))) {
|
||||
return WORSE_RESULT;
|
||||
}
|
||||
|
||||
CBENCHMARK(loopDurationC < TIMELOOP_NANOSEC, benchres, tmp, BMK_compressOnly, BMK_timeMode, 1);
|
||||
CBENCHMARK(loopDurationD < TIMELOOP_NANOSEC, benchres, tmp, BMK_decodeOnly, BMK_timeMode, 1);
|
||||
|
||||
*resultPtr = benchres;
|
||||
|
||||
/* compare by resultScore when in infeas */
|
||||
/* compare by compareResultLT when in feas */
|
||||
if((!feas && (resultScore(benchres, buf.srcSize, target) > resultScore(*winnerResult, buf.srcSize, target))) ||
|
||||
(feas && (compareResultLT(*winnerResult, benchres, target, buf.srcSize))) ) {
|
||||
return BETTER_RESULT;
|
||||
} else {
|
||||
return WORSE_RESULT;
|
||||
}
|
||||
}
|
||||
|
||||
#define INFEASIBLE_THRESHOLD 200
|
||||
/* Memoized benchmarking, won't benchmark anything which has already been benchmarked before. */
|
||||
static int benchMemo(BMK_result_t* resultPtr,
|
||||
const buffers_t buf, const contexts_t ctx,
|
||||
const paramValues_t cParams,
|
||||
const constraint_t target,
|
||||
BMK_result_t* winnerResult, memoTable_t* const memoTableArray,
|
||||
const int feas) {
|
||||
static int bmcount = 0;
|
||||
int res;
|
||||
|
||||
if(memoTableGet(memoTableArray, cParams) >= INFEASIBLE_THRESHOLD || redundantParams(cParams, target, buf.maxBlockSize)) { return WORSE_RESULT; }
|
||||
|
||||
res = allBench(resultPtr, buf, ctx, cParams, target, winnerResult, feas);
|
||||
|
||||
if(DEBUG && !(bmcount % 250)) {
|
||||
DISPLAY("Count: %d\n", bmcount);
|
||||
bmcount++;
|
||||
}
|
||||
BMK_printWinnerOpt(stdout, CUSTOM_LEVEL, *resultPtr, cParams, target, buf.srcSize);
|
||||
|
||||
if(res == BETTER_RESULT || feas) {
|
||||
memoTableSet(memoTableArray, cParams, 255); /* what happens if collisions are frequent */
|
||||
}
|
||||
return res;
|
||||
}
|
||||
|
||||
typedef struct {
|
||||
U64 cSpeed_min;
|
||||
U64 dSpeed_min;
|
||||
U32 windowLog_max;
|
||||
ZSTD_strategy strategy_max;
|
||||
} level_constraints_t;
|
||||
@@ -243,15 +1659,14 @@ static void BMK_init_level_constraints(int bytePerSec_level1)
|
||||
} }
|
||||
}
|
||||
|
||||
static int BMK_seed(winnerInfo_t* winners, const ZSTD_compressionParameters params,
|
||||
const void* srcBuffer, size_t srcSize,
|
||||
ZSTD_CCtx* ctx, ZSTD_DCtx* dctx)
|
||||
static int BMK_seed(winnerInfo_t* winners, const paramValues_t params,
|
||||
const buffers_t buf, const contexts_t ctx)
|
||||
{
|
||||
BMK_result_t testResult;
|
||||
int better = 0;
|
||||
int cLevel;
|
||||
|
||||
BMK_benchParam(&testResult, srcBuffer, srcSize, ctx, dctx, params);
|
||||
BMK_benchParam(&testResult, buf, ctx, params);
|
||||
|
||||
|
||||
for (cLevel = 1; cLevel <= NB_LEVELS_TRACKED; cLevel++) {
|
||||
@@ -259,32 +1674,32 @@ static int BMK_seed(winnerInfo_t* winners, const ZSTD_compressionParameters para
|
||||
continue; /* not fast enough for this level */
|
||||
if (testResult.dSpeed < g_level_constraint[cLevel].dSpeed_min)
|
||||
continue; /* not fast enough for this level */
|
||||
if (params.windowLog > g_level_constraint[cLevel].windowLog_max)
|
||||
if (params.vals[wlog_ind] > g_level_constraint[cLevel].windowLog_max)
|
||||
continue; /* too much memory for this level */
|
||||
if (params.strategy > g_level_constraint[cLevel].strategy_max)
|
||||
if (params.vals[strt_ind] > g_level_constraint[cLevel].strategy_max)
|
||||
continue; /* forbidden strategy for this level */
|
||||
if (winners[cLevel].result.cSize==0) {
|
||||
/* first solution for this cLevel */
|
||||
winners[cLevel].result = testResult;
|
||||
winners[cLevel].params = params;
|
||||
BMK_printWinner(stdout, cLevel, testResult, params, srcSize);
|
||||
BMK_printWinner(stdout, cLevel, testResult, params, buf.srcSize);
|
||||
better = 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
if ((double)testResult.cSize <= ((double)winners[cLevel].result.cSize * (1. + (0.02 / cLevel))) ) {
|
||||
/* Validate solution is "good enough" */
|
||||
double W_ratio = (double)srcSize / testResult.cSize;
|
||||
double O_ratio = (double)srcSize / winners[cLevel].result.cSize;
|
||||
double W_ratio = (double)buf.srcSize / testResult.cSize;
|
||||
double O_ratio = (double)buf.srcSize / winners[cLevel].result.cSize;
|
||||
double W_ratioNote = log (W_ratio);
|
||||
double O_ratioNote = log (O_ratio);
|
||||
size_t W_DMemUsed = (1 << params.windowLog) + (16 KB);
|
||||
size_t O_DMemUsed = (1 << winners[cLevel].params.windowLog) + (16 KB);
|
||||
size_t W_DMemUsed = (1 << params.vals[wlog_ind]) + (16 KB);
|
||||
size_t O_DMemUsed = (1 << winners[cLevel].params.vals[wlog_ind]) + (16 KB);
|
||||
double W_DMemUsed_note = W_ratioNote * ( 40 + 9*cLevel) - log((double)W_DMemUsed);
|
||||
double O_DMemUsed_note = O_ratioNote * ( 40 + 9*cLevel) - log((double)O_DMemUsed);
|
||||
|
||||
size_t W_CMemUsed = (1 << params.windowLog) + ZSTD_estimateCCtxSize_usingCParams(params);
|
||||
size_t O_CMemUsed = (1 << winners[cLevel].params.windowLog) + ZSTD_estimateCCtxSize_usingCParams(winners[cLevel].params);
|
||||
size_t W_CMemUsed = (1 << params.vals[wlog_ind]) + ZSTD_estimateCCtxSize_usingCParams(pvalsToCParams(params));
|
||||
size_t O_CMemUsed = (1 << winners[cLevel].params.vals[wlog_ind]) + ZSTD_estimateCCtxSize_usingCParams(pvalsToCParams(winners[cLevel].params));
|
||||
double W_CMemUsed_note = W_ratioNote * ( 50 + 13*cLevel) - log((double)W_CMemUsed);
|
||||
double O_CMemUsed_note = O_ratioNote * ( 50 + 13*cLevel) - log((double)O_CMemUsed);
|
||||
|
||||
@@ -314,16 +1729,16 @@ static int BMK_seed(winnerInfo_t* winners, const ZSTD_compressionParameters para
|
||||
/* too large compression speed difference for the compression benefit */
|
||||
if (W_ratio > O_ratio)
|
||||
DISPLAY ("Compression Speed : %5.3f @ %4.1f MB/s vs %5.3f @ %4.1f MB/s : not enough for level %i\n",
|
||||
W_ratio, testResult.cSpeed / 1000000,
|
||||
O_ratio, winners[cLevel].result.cSpeed / 1000000., cLevel);
|
||||
W_ratio, (double)testResult.cSpeed / (1 MB),
|
||||
O_ratio, (double)winners[cLevel].result.cSpeed / (1 MB), cLevel);
|
||||
continue;
|
||||
}
|
||||
if (W_DSpeed_note < O_DSpeed_note ) {
|
||||
/* too large decompression speed difference for the compression benefit */
|
||||
if (W_ratio > O_ratio)
|
||||
DISPLAY ("Decompression Speed : %5.3f @ %4.1f MB/s vs %5.3f @ %4.1f MB/s : not enough for level %i\n",
|
||||
W_ratio, testResult.dSpeed / 1000000.,
|
||||
O_ratio, winners[cLevel].result.dSpeed / 1000000., cLevel);
|
||||
W_ratio, (double)testResult.dSpeed / (1 MB),
|
||||
O_ratio, (double)winners[cLevel].result.dSpeed / (1 MB), cLevel);
|
||||
continue;
|
||||
}
|
||||
|
||||
@@ -332,7 +1747,7 @@ static int BMK_seed(winnerInfo_t* winners, const ZSTD_compressionParameters para
|
||||
|
||||
winners[cLevel].result = testResult;
|
||||
winners[cLevel].params = params;
|
||||
BMK_printWinner(stdout, cLevel, testResult, params, srcSize);
|
||||
BMK_printWinner(stdout, cLevel, testResult, params, buf.srcSize);
|
||||
|
||||
better = 1;
|
||||
} }
|
||||
@@ -340,158 +1755,72 @@ static int BMK_seed(winnerInfo_t* winners, const ZSTD_compressionParameters para
|
||||
return better;
|
||||
}
|
||||
|
||||
|
||||
/* nullified useless params, to ensure count stats */
|
||||
static ZSTD_compressionParameters* sanitizeParams(ZSTD_compressionParameters params)
|
||||
{
|
||||
g_params = params;
|
||||
if (params.strategy == ZSTD_fast)
|
||||
g_params.chainLog = 0, g_params.searchLog = 0;
|
||||
if (params.strategy == ZSTD_dfast)
|
||||
g_params.searchLog = 0;
|
||||
if (params.strategy != ZSTD_btopt && params.strategy != ZSTD_btultra)
|
||||
g_params.targetLength = 0;
|
||||
return &g_params;
|
||||
}
|
||||
|
||||
|
||||
static void paramVariation(ZSTD_compressionParameters* ptr)
|
||||
{
|
||||
ZSTD_compressionParameters p;
|
||||
U32 validated = 0;
|
||||
while (!validated) {
|
||||
U32 nbChanges = (FUZ_rand(&g_rand) & 3) + 1;
|
||||
p = *ptr;
|
||||
for ( ; nbChanges ; nbChanges--) {
|
||||
const U32 changeID = FUZ_rand(&g_rand) % 14;
|
||||
switch(changeID)
|
||||
{
|
||||
case 0:
|
||||
p.chainLog++; break;
|
||||
case 1:
|
||||
p.chainLog--; break;
|
||||
case 2:
|
||||
p.hashLog++; break;
|
||||
case 3:
|
||||
p.hashLog--; break;
|
||||
case 4:
|
||||
p.searchLog++; break;
|
||||
case 5:
|
||||
p.searchLog--; break;
|
||||
case 6:
|
||||
p.windowLog++; break;
|
||||
case 7:
|
||||
p.windowLog--; break;
|
||||
case 8:
|
||||
p.searchLength++; break;
|
||||
case 9:
|
||||
p.searchLength--; break;
|
||||
case 10:
|
||||
p.strategy = (ZSTD_strategy)(((U32)p.strategy)+1); break;
|
||||
case 11:
|
||||
p.strategy = (ZSTD_strategy)(((U32)p.strategy)-1); break;
|
||||
case 12:
|
||||
p.targetLength *= 1 + ((double)(FUZ_rand(&g_rand)&255)) / 256.; break;
|
||||
case 13:
|
||||
p.targetLength /= 1 + ((double)(FUZ_rand(&g_rand)&255)) / 256.; break;
|
||||
}
|
||||
}
|
||||
validated = !ZSTD_isError(ZSTD_checkCParams(p));
|
||||
}
|
||||
*ptr = p;
|
||||
}
|
||||
|
||||
/*-************************************
|
||||
* Compression Level Table Generation Functions
|
||||
**************************************/
|
||||
|
||||
#define PARAMTABLELOG 25
|
||||
#define PARAMTABLESIZE (1<<PARAMTABLELOG)
|
||||
#define PARAMTABLEMASK (PARAMTABLESIZE-1)
|
||||
static BYTE g_alreadyTested[PARAMTABLESIZE] = {0}; /* init to zero */
|
||||
|
||||
#define NB_TESTS_PLAYED(p) \
|
||||
g_alreadyTested[(XXH64(sanitizeParams(p), sizeof(p), 0) >> 3) & PARAMTABLEMASK]
|
||||
|
||||
static BYTE* NB_TESTS_PLAYED(paramValues_t p) {
|
||||
ZSTD_compressionParameters p2 = pvalsToCParams(sanitizeParams(p));
|
||||
return &g_alreadyTested[(XXH64((void*)&p2, sizeof(p2), 0) >> 3) & PARAMTABLEMASK];
|
||||
}
|
||||
|
||||
static void playAround(FILE* f, winnerInfo_t* winners,
|
||||
ZSTD_compressionParameters params,
|
||||
const void* srcBuffer, size_t srcSize,
|
||||
ZSTD_CCtx* ctx, ZSTD_DCtx* dctx)
|
||||
paramValues_t p,
|
||||
const buffers_t buf, const contexts_t ctx)
|
||||
{
|
||||
int nbVariations = 0;
|
||||
int nbVariations = 0, i;
|
||||
UTIL_time_t const clockStart = UTIL_getTime();
|
||||
|
||||
while (UTIL_clockSpanMicro(clockStart) < g_maxVariationTime) {
|
||||
ZSTD_compressionParameters p = params;
|
||||
BYTE* b;
|
||||
|
||||
if (nbVariations++ > g_maxNbVariations) break;
|
||||
paramVariation(&p);
|
||||
|
||||
do { for(i = 0; i < 4; i++) { paramVaryOnce(FUZ_rand(&g_rand) % (strt_ind + 1), ((FUZ_rand(&g_rand) & 1) << 1) - 1, &p); } }
|
||||
while(!paramValid(p));
|
||||
|
||||
/* exclude faster if already played params */
|
||||
if (FUZ_rand(&g_rand) & ((1 << NB_TESTS_PLAYED(p))-1))
|
||||
if (FUZ_rand(&g_rand) & ((1 << *NB_TESTS_PLAYED(p))-1))
|
||||
continue;
|
||||
|
||||
/* test */
|
||||
NB_TESTS_PLAYED(p)++;
|
||||
if (!BMK_seed(winners, p, srcBuffer, srcSize, ctx, dctx)) continue;
|
||||
b = NB_TESTS_PLAYED(p);
|
||||
(*b)++;
|
||||
if (!BMK_seed(winners, p, buf, ctx)) continue;
|
||||
|
||||
/* improvement found => search more */
|
||||
BMK_printWinners(f, winners, srcSize);
|
||||
playAround(f, winners, p, srcBuffer, srcSize, ctx, dctx);
|
||||
BMK_printWinners(f, winners, buf.srcSize);
|
||||
playAround(f, winners, p, buf, ctx);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
static ZSTD_compressionParameters randomParams(void)
|
||||
{
|
||||
ZSTD_compressionParameters p;
|
||||
U32 validated = 0;
|
||||
while (!validated) {
|
||||
/* totally random entry */
|
||||
p.chainLog = (FUZ_rand(&g_rand) % (ZSTD_CHAINLOG_MAX+1 - ZSTD_CHAINLOG_MIN)) + ZSTD_CHAINLOG_MIN;
|
||||
p.hashLog = (FUZ_rand(&g_rand) % (ZSTD_HASHLOG_MAX+1 - ZSTD_HASHLOG_MIN)) + ZSTD_HASHLOG_MIN;
|
||||
p.searchLog = (FUZ_rand(&g_rand) % (ZSTD_SEARCHLOG_MAX+1 - ZSTD_SEARCHLOG_MIN)) + ZSTD_SEARCHLOG_MIN;
|
||||
p.windowLog = (FUZ_rand(&g_rand) % (ZSTD_WINDOWLOG_MAX+1 - ZSTD_WINDOWLOG_MIN)) + ZSTD_WINDOWLOG_MIN;
|
||||
p.searchLength=(FUZ_rand(&g_rand) % (ZSTD_SEARCHLENGTH_MAX+1 - ZSTD_SEARCHLENGTH_MIN)) + ZSTD_SEARCHLENGTH_MIN;
|
||||
p.targetLength=(FUZ_rand(&g_rand) % (512));
|
||||
p.strategy = (ZSTD_strategy) (FUZ_rand(&g_rand) % (ZSTD_btultra +1));
|
||||
validated = !ZSTD_isError(ZSTD_checkCParams(p));
|
||||
}
|
||||
return p;
|
||||
}
|
||||
|
||||
static void BMK_selectRandomStart(
|
||||
FILE* f, winnerInfo_t* winners,
|
||||
const void* srcBuffer, size_t srcSize,
|
||||
ZSTD_CCtx* ctx, ZSTD_DCtx* dctx)
|
||||
const buffers_t buf, const contexts_t ctx)
|
||||
{
|
||||
U32 const id = FUZ_rand(&g_rand) % (NB_LEVELS_TRACKED+1);
|
||||
if ((id==0) || (winners[id].params.windowLog==0)) {
|
||||
if ((id==0) || (winners[id].params.vals[wlog_ind]==0)) {
|
||||
/* use some random entry */
|
||||
ZSTD_compressionParameters const p = ZSTD_adjustCParams(randomParams(), srcSize, 0);
|
||||
playAround(f, winners, p, srcBuffer, srcSize, ctx, dctx);
|
||||
paramValues_t const p = adjustParams(cParamsToPVals(pvalsToCParams(randomParams())), /* defaults nonCompression parameters */
|
||||
buf.srcSize, 0);
|
||||
playAround(f, winners, p, buf, ctx);
|
||||
} else {
|
||||
playAround(f, winners, winners[id].params, srcBuffer, srcSize, ctx, dctx);
|
||||
playAround(f, winners, winners[id].params, buf, ctx);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
static void BMK_benchOnce(ZSTD_CCtx* cctx, ZSTD_DCtx* dctx, const void* srcBuffer, size_t srcSize)
|
||||
static void BMK_benchFullTable(const buffers_t buf, const contexts_t ctx)
|
||||
{
|
||||
BMK_result_t testResult;
|
||||
g_params = ZSTD_adjustCParams(g_params, srcSize, 0);
|
||||
BMK_benchParam(&testResult, srcBuffer, srcSize, cctx, dctx, g_params);
|
||||
DISPLAY("Compression Ratio: %.3f Compress Speed: %.1f MB/s Decompress Speed: %.1f MB/s\n", (double)srcSize / testResult.cSize,
|
||||
testResult.cSpeed / 1000000, testResult.dSpeed / 1000000);
|
||||
return;
|
||||
}
|
||||
|
||||
static void BMK_benchFullTable(ZSTD_CCtx* cctx, ZSTD_DCtx* dctx, const void* srcBuffer, size_t srcSize)
|
||||
{
|
||||
ZSTD_compressionParameters params;
|
||||
paramValues_t params;
|
||||
winnerInfo_t winners[NB_LEVELS_TRACKED+1];
|
||||
const char* const rfName = "grillResults.txt";
|
||||
FILE* const f = fopen(rfName, "w");
|
||||
const size_t blockSize = g_blockSize ? g_blockSize : srcSize; /* cut by block or not ? */
|
||||
|
||||
/* init */
|
||||
assert(g_singleRun==0);
|
||||
@@ -499,12 +1828,12 @@ static void BMK_benchFullTable(ZSTD_CCtx* cctx, ZSTD_DCtx* dctx, const void* src
|
||||
if (f==NULL) { DISPLAY("error opening %s \n", rfName); exit(1); }
|
||||
|
||||
if (g_target) {
|
||||
BMK_init_level_constraints(g_target*1000000);
|
||||
BMK_init_level_constraints(g_target * (1 MB));
|
||||
} else {
|
||||
/* baseline config for level 1 */
|
||||
ZSTD_compressionParameters const l1params = ZSTD_getCParams(1, blockSize, 0);
|
||||
paramValues_t const l1params = cParamsToPVals(ZSTD_getCParams(1, buf.maxBlockSize, ctx.dictSize));
|
||||
BMK_result_t testResult;
|
||||
BMK_benchParam(&testResult, srcBuffer, srcSize, cctx, dctx, l1params);
|
||||
BMK_benchParam(&testResult, buf, ctx, l1params);
|
||||
BMK_init_level_constraints((int)((testResult.cSpeed * 31) / 32));
|
||||
}
|
||||
|
||||
@@ -512,251 +1841,543 @@ static void BMK_benchFullTable(ZSTD_CCtx* cctx, ZSTD_DCtx* dctx, const void* src
|
||||
{ const int maxSeeds = g_noSeed ? 1 : ZSTD_maxCLevel();
|
||||
int i;
|
||||
for (i=0; i<=maxSeeds; i++) {
|
||||
params = ZSTD_getCParams(i, blockSize, 0);
|
||||
BMK_seed(winners, params, srcBuffer, srcSize, cctx, dctx);
|
||||
params = cParamsToPVals(ZSTD_getCParams(i, buf.maxBlockSize, 0));
|
||||
BMK_seed(winners, params, buf, ctx);
|
||||
} }
|
||||
BMK_printWinners(f, winners, srcSize);
|
||||
BMK_printWinners(f, winners, buf.srcSize);
|
||||
|
||||
/* start tests */
|
||||
{ const time_t grillStart = time(NULL);
|
||||
{ const UTIL_time_t grillStart = UTIL_getTime();
|
||||
do {
|
||||
BMK_selectRandomStart(f, winners, srcBuffer, srcSize, cctx, dctx);
|
||||
} while (BMK_timeSpan(grillStart) < g_grillDuration_s);
|
||||
BMK_selectRandomStart(f, winners, buf, ctx);
|
||||
} while (BMK_timeSpan(grillStart) < g_timeLimit_s);
|
||||
}
|
||||
|
||||
/* end summary */
|
||||
BMK_printWinners(f, winners, srcSize);
|
||||
BMK_printWinners(f, winners, buf.srcSize);
|
||||
DISPLAY("grillParams operations completed \n");
|
||||
|
||||
/* clean up*/
|
||||
fclose(f);
|
||||
}
|
||||
|
||||
static void BMK_benchMem_usingCCtx(ZSTD_CCtx* const cctx, ZSTD_DCtx* const dctx, const void* srcBuffer, size_t srcSize)
|
||||
{
|
||||
if (g_singleRun)
|
||||
return BMK_benchOnce(cctx, dctx, srcBuffer, srcSize);
|
||||
else
|
||||
return BMK_benchFullTable(cctx, dctx, srcBuffer, srcSize);
|
||||
}
|
||||
/*-************************************
|
||||
* Single Benchmark Functions
|
||||
**************************************/
|
||||
|
||||
static void BMK_benchMemCCtxInit(const void* srcBuffer, size_t srcSize)
|
||||
{
|
||||
ZSTD_CCtx* const cctx = ZSTD_createCCtx();
|
||||
ZSTD_DCtx* const dctx = ZSTD_createDCtx();
|
||||
if (cctx==NULL || dctx==NULL) { DISPLAY("Context Creation failed \n"); exit(1); }
|
||||
BMK_benchMem_usingCCtx(cctx, dctx, srcBuffer, srcSize);
|
||||
ZSTD_freeCCtx(cctx);
|
||||
}
|
||||
static int benchOnce(const buffers_t buf, const contexts_t ctx, const int cLevel) {
|
||||
BMK_result_t testResult;
|
||||
g_params = adjustParams(overwriteParams(cParamsToPVals(ZSTD_getCParams(cLevel, buf.maxBlockSize, ctx.dictSize)), g_params), buf.maxBlockSize, ctx.dictSize);
|
||||
|
||||
if(BMK_benchParam(&testResult, buf, ctx, g_params)) {
|
||||
DISPLAY("Error during benchmarking\n");
|
||||
return 1;
|
||||
}
|
||||
|
||||
static int benchSample(void)
|
||||
{
|
||||
const char* const name = "Sample 10MB";
|
||||
size_t const benchedSize = 10000000;
|
||||
BMK_printWinner(stdout, CUSTOM_LEVEL, testResult, g_params, buf.srcSize);
|
||||
|
||||
void* origBuff = malloc(benchedSize);
|
||||
if (!origBuff) { perror("not enough memory"); return 12; }
|
||||
|
||||
/* Fill buffer */
|
||||
RDG_genBuffer(origBuff, benchedSize, g_compressibility, 0.0, 0);
|
||||
|
||||
/* bench */
|
||||
DISPLAY("\r%79s\r", "");
|
||||
DISPLAY("using %s %i%%: \n", name, (int)(g_compressibility*100));
|
||||
BMK_benchMemCCtxInit(origBuff, benchedSize);
|
||||
|
||||
free(origBuff);
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int benchSample(double compressibility, int cLevel)
|
||||
{
|
||||
const char* const name = "Sample 10MB";
|
||||
size_t const benchedSize = 10 MB;
|
||||
void* srcBuffer = malloc(benchedSize);
|
||||
int ret = 0;
|
||||
|
||||
buffers_t buf;
|
||||
contexts_t ctx;
|
||||
|
||||
if(srcBuffer == NULL) {
|
||||
DISPLAY("Out of Memory\n");
|
||||
return 2;
|
||||
}
|
||||
|
||||
RDG_genBuffer(srcBuffer, benchedSize, compressibility, 0.0, 0);
|
||||
|
||||
if(createBuffersFromMemory(&buf, srcBuffer, 1, &benchedSize)) {
|
||||
DISPLAY("Buffer Creation Error\n");
|
||||
free(srcBuffer);
|
||||
return 3;
|
||||
}
|
||||
|
||||
if(createContexts(&ctx, NULL)) {
|
||||
DISPLAY("Context Creation Error\n");
|
||||
freeBuffers(buf);
|
||||
return 1;
|
||||
}
|
||||
|
||||
/* bench */
|
||||
DISPLAY("\r%79s\r", "");
|
||||
DISPLAY("using %s %i%%: \n", name, (int)(compressibility*100));
|
||||
|
||||
if(g_singleRun) {
|
||||
ret = benchOnce(buf, ctx, cLevel);
|
||||
} else {
|
||||
BMK_benchFullTable(buf, ctx);
|
||||
}
|
||||
|
||||
freeBuffers(buf);
|
||||
freeContexts(ctx);
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
/* benchFiles() :
|
||||
* note: while this function takes a table of filenames,
|
||||
* in practice, only the first filename will be used */
|
||||
int benchFiles(const char** fileNamesTable, int nbFiles)
|
||||
int benchFiles(const char** fileNamesTable, int nbFiles, const char* dictFileName, const int cLevel)
|
||||
{
|
||||
int fileIdx=0;
|
||||
buffers_t buf;
|
||||
contexts_t ctx;
|
||||
int ret = 0;
|
||||
|
||||
/* Loop for each file */
|
||||
while (fileIdx<nbFiles) {
|
||||
const char* const inFileName = fileNamesTable[fileIdx++];
|
||||
FILE* const inFile = fopen( inFileName, "rb" );
|
||||
U64 const inFileSize = UTIL_getFileSize(inFileName);
|
||||
size_t benchedSize;
|
||||
void* origBuff;
|
||||
|
||||
/* Check file existence */
|
||||
if (inFile==NULL) {
|
||||
DISPLAY( "Pb opening %s\n", inFileName);
|
||||
return 11;
|
||||
}
|
||||
if (inFileSize == UTIL_FILESIZE_UNKNOWN) {
|
||||
DISPLAY("Pb evaluating size of %s \n", inFileName);
|
||||
fclose(inFile);
|
||||
return 11;
|
||||
}
|
||||
|
||||
/* Memory allocation */
|
||||
benchedSize = BMK_findMaxMem(inFileSize*3) / 3;
|
||||
if ((U64)benchedSize > inFileSize) benchedSize = (size_t)inFileSize;
|
||||
if (benchedSize < inFileSize)
|
||||
DISPLAY("Not enough memory for '%s' full size; testing %i MB only...\n", inFileName, (int)(benchedSize>>20));
|
||||
origBuff = malloc(benchedSize);
|
||||
if (origBuff==NULL) {
|
||||
DISPLAY("\nError: not enough memory!\n");
|
||||
fclose(inFile);
|
||||
return 12;
|
||||
}
|
||||
|
||||
/* Fill input buffer */
|
||||
DISPLAY("Loading %s... \r", inFileName);
|
||||
{ size_t const readSize = fread(origBuff, 1, benchedSize, inFile);
|
||||
fclose(inFile);
|
||||
if(readSize != benchedSize) {
|
||||
DISPLAY("\nError: problem reading file '%s' !! \n", inFileName);
|
||||
free(origBuff);
|
||||
return 13;
|
||||
} }
|
||||
|
||||
/* bench */
|
||||
DISPLAY("\r%79s\r", "");
|
||||
DISPLAY("using %s : \n", inFileName);
|
||||
BMK_benchMemCCtxInit(origBuff, benchedSize);
|
||||
|
||||
/* clean */
|
||||
free(origBuff);
|
||||
if(createBuffers(&buf, fileNamesTable, nbFiles)) {
|
||||
DISPLAY("unable to load files\n");
|
||||
return 1;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
static void BMK_translateAdvancedParams(ZSTD_compressionParameters params)
|
||||
{
|
||||
DISPLAY("--zstd=windowLog=%u,chainLog=%u,hashLog=%u,searchLog=%u,searchLength=%u,targetLength=%u,strategy=%u \n",
|
||||
params.windowLog, params.chainLog, params.hashLog, params.searchLog, params.searchLength, params.targetLength, (U32)(params.strategy));
|
||||
}
|
||||
|
||||
/* optimizeForSize():
|
||||
* targetSpeed : expressed in MB/s */
|
||||
int optimizeForSize(const char* inFileName, U32 targetSpeed)
|
||||
{
|
||||
FILE* const inFile = fopen( inFileName, "rb" );
|
||||
U64 const inFileSize = UTIL_getFileSize(inFileName);
|
||||
size_t benchedSize = BMK_findMaxMem(inFileSize*3) / 3;
|
||||
void* origBuff;
|
||||
/* Init */
|
||||
if (inFile==NULL) { DISPLAY( "Pb opening %s\n", inFileName); return 11; }
|
||||
if (inFileSize == UTIL_FILESIZE_UNKNOWN) {
|
||||
DISPLAY("Pb evaluatin size of %s \n", inFileName);
|
||||
fclose(inFile);
|
||||
return 11;
|
||||
if(createContexts(&ctx, dictFileName)) {
|
||||
DISPLAY("unable to load dictionary\n");
|
||||
freeBuffers(buf);
|
||||
return 2;
|
||||
}
|
||||
|
||||
/* Memory allocation & restrictions */
|
||||
if ((U64)benchedSize > inFileSize) benchedSize = (size_t)inFileSize;
|
||||
if (benchedSize < inFileSize) {
|
||||
DISPLAY("Not enough memory for '%s' \n", inFileName);
|
||||
fclose(inFile);
|
||||
return 11;
|
||||
}
|
||||
|
||||
/* Alloc */
|
||||
origBuff = malloc(benchedSize);
|
||||
if(!origBuff) {
|
||||
DISPLAY("\nError: not enough memory!\n");
|
||||
fclose(inFile);
|
||||
return 12;
|
||||
}
|
||||
|
||||
/* Fill input buffer */
|
||||
DISPLAY("Loading %s... \r", inFileName);
|
||||
{ size_t const readSize = fread(origBuff, 1, benchedSize, inFile);
|
||||
fclose(inFile);
|
||||
if(readSize != benchedSize) {
|
||||
DISPLAY("\nError: problem reading file '%s' !! \n", inFileName);
|
||||
free(origBuff);
|
||||
return 13;
|
||||
} }
|
||||
|
||||
/* bench */
|
||||
DISPLAY("\r%79s\r", "");
|
||||
DISPLAY("optimizing for %s - limit speed %u MB/s \n", inFileName, targetSpeed);
|
||||
targetSpeed *= 1000000;
|
||||
{ ZSTD_CCtx* const ctx = ZSTD_createCCtx();
|
||||
ZSTD_DCtx* const dctx = ZSTD_createDCtx();
|
||||
winnerInfo_t winner;
|
||||
BMK_result_t candidate;
|
||||
const size_t blockSize = g_blockSize ? g_blockSize : benchedSize;
|
||||
if(nbFiles == 1) {
|
||||
DISPLAY("using %s : \n", fileNamesTable[0]);
|
||||
} else {
|
||||
DISPLAY("using %d Files : \n", nbFiles);
|
||||
}
|
||||
|
||||
/* init */
|
||||
if (ctx==NULL) { DISPLAY("\n ZSTD_createCCtx error \n"); free(origBuff); return 14;}
|
||||
memset(&winner, 0, sizeof(winner));
|
||||
winner.result.cSize = (size_t)(-1);
|
||||
if(g_singleRun) {
|
||||
ret = benchOnce(buf, ctx, cLevel);
|
||||
} else {
|
||||
BMK_benchFullTable(buf, ctx);
|
||||
}
|
||||
|
||||
/* find best solution from default params */
|
||||
{ const int maxSeeds = g_noSeed ? 1 : ZSTD_maxCLevel();
|
||||
int i;
|
||||
for (i=1; i<=maxSeeds; i++) {
|
||||
ZSTD_compressionParameters const CParams = ZSTD_getCParams(i, blockSize, 0);
|
||||
BMK_benchParam(&candidate, origBuff, benchedSize, ctx, dctx, CParams);
|
||||
if (candidate.cSpeed < (double)targetSpeed) {
|
||||
freeBuffers(buf);
|
||||
freeContexts(ctx);
|
||||
return ret;
|
||||
}
|
||||
|
||||
|
||||
/*-************************************
|
||||
* Local Optimization Functions
|
||||
**************************************/
|
||||
|
||||
/* One iteration of hill climbing. Specifically, it first tries all
|
||||
* valid parameter configurations w/ manhattan distance 1 and picks the best one
|
||||
* failing that, it progressively tries candidates further and further away (up to #dim + 2)
|
||||
* if it finds a candidate exceeding winnerInfo, it will repeat. Otherwise, it will stop the
|
||||
* current stage of hill climbing.
|
||||
* Each iteration of hill climbing proceeds in 2 'phases'. Phase 1 climbs according to
|
||||
* the resultScore function, which is effectively a linear increase in reward until it reaches
|
||||
* the constraint-satisfying value, it which point any excess results in only logarithmic reward.
|
||||
* This aims to find some constraint-satisfying point.
|
||||
* Phase 2 optimizes in accordance with what the original function sets out to maximize, with
|
||||
* all feasible solutions valued over all infeasible solutions.
|
||||
*/
|
||||
|
||||
/* sanitize all params here.
|
||||
* all generation after random should be sanitized. (maybe sanitize random)
|
||||
*/
|
||||
static winnerInfo_t climbOnce(const constraint_t target,
|
||||
memoTable_t* mtAll,
|
||||
const buffers_t buf, const contexts_t ctx,
|
||||
const paramValues_t init) {
|
||||
/*
|
||||
* cparam - currently considered 'center'
|
||||
* candidate - params to benchmark/results
|
||||
* winner - best option found so far.
|
||||
*/
|
||||
paramValues_t cparam = init;
|
||||
winnerInfo_t candidateInfo, winnerInfo;
|
||||
int better = 1;
|
||||
int feas = 0;
|
||||
|
||||
winnerInfo = initWinnerInfo(init);
|
||||
candidateInfo = winnerInfo;
|
||||
|
||||
{
|
||||
winnerInfo_t bestFeasible1 = initWinnerInfo(cparam);
|
||||
DEBUGOUTPUT("Climb Part 1\n");
|
||||
while(better) {
|
||||
|
||||
int offset;
|
||||
size_t i, dist;
|
||||
const size_t varLen = mtAll[cparam.vals[strt_ind]].varLen;
|
||||
better = 0;
|
||||
DEBUGOUTPUT("Start\n");
|
||||
cparam = winnerInfo.params;
|
||||
candidateInfo.params = cparam;
|
||||
/* all dist-1 candidates */
|
||||
for(i = 0; i < varLen; i++) {
|
||||
for(offset = -1; offset <= 1; offset += 2) {
|
||||
CHECKTIME(winnerInfo);
|
||||
candidateInfo.params = cparam;
|
||||
paramVaryOnce(mtAll[cparam.vals[strt_ind]].varArray[i], offset, &candidateInfo.params);
|
||||
|
||||
if(paramValid(candidateInfo.params)) {
|
||||
int res;
|
||||
res = benchMemo(&candidateInfo.result, buf, ctx,
|
||||
sanitizeParams(candidateInfo.params), target, &winnerInfo.result, mtAll, feas);
|
||||
DEBUGOUTPUT("Res: %d\n", res);
|
||||
if(res == BETTER_RESULT) { /* synonymous with better when called w/ infeasibleBM */
|
||||
winnerInfo = candidateInfo;
|
||||
better = 1;
|
||||
if(compareResultLT(bestFeasible1.result, winnerInfo.result, target, buf.srcSize)) {
|
||||
bestFeasible1 = winnerInfo;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if(better) {
|
||||
continue;
|
||||
}
|
||||
|
||||
for(dist = 2; dist < varLen + 2; dist++) { /* varLen is # dimensions */
|
||||
for(i = 0; i < (1 << varLen) / varLen + 2; i++) {
|
||||
int res;
|
||||
CHECKTIME(winnerInfo);
|
||||
candidateInfo.params = cparam;
|
||||
/* param error checking already done here */
|
||||
paramVariation(&candidateInfo.params, mtAll, (U32)dist);
|
||||
|
||||
res = benchMemo(&candidateInfo.result, buf, ctx,
|
||||
sanitizeParams(candidateInfo.params), target, &winnerInfo.result, mtAll, feas);
|
||||
DEBUGOUTPUT("Res: %d\n", res);
|
||||
if(res == BETTER_RESULT) { /* synonymous with better in this case*/
|
||||
winnerInfo = candidateInfo;
|
||||
better = 1;
|
||||
if(compareResultLT(bestFeasible1.result, winnerInfo.result, target, buf.srcSize)) {
|
||||
bestFeasible1 = winnerInfo;
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
}
|
||||
if(better) {
|
||||
break;
|
||||
}
|
||||
if ( (candidate.cSize < winner.result.cSize)
|
||||
| ((candidate.cSize == winner.result.cSize) & (candidate.cSpeed > winner.result.cSpeed)) )
|
||||
{
|
||||
winner.params = CParams;
|
||||
winner.result = candidate;
|
||||
BMK_printWinner(stdout, i, winner.result, winner.params, benchedSize);
|
||||
} }
|
||||
}
|
||||
|
||||
if(!better) { /* infeas -> feas -> stop */
|
||||
if(feas) { return winnerInfo; }
|
||||
|
||||
feas = 1;
|
||||
better = 1;
|
||||
winnerInfo = bestFeasible1; /* note with change, bestFeasible may not necessarily be feasible, but if one has been benchmarked, it will be. */
|
||||
DEBUGOUTPUT("Climb Part 2\n");
|
||||
}
|
||||
}
|
||||
|
||||
BMK_printWinner(stdout, CUSTOM_LEVEL, winner.result, winner.params, benchedSize);
|
||||
|
||||
BMK_translateAdvancedParams(winner.params);
|
||||
|
||||
/* start tests */
|
||||
{ time_t const grillStart = time(NULL);
|
||||
do {
|
||||
ZSTD_compressionParameters params = winner.params;
|
||||
paramVariation(¶ms);
|
||||
if ((FUZ_rand(&g_rand) & 31) == 3) params = randomParams(); /* totally random config to improve search space */
|
||||
params = ZSTD_adjustCParams(params, blockSize, 0);
|
||||
|
||||
/* exclude faster if already played set of params */
|
||||
if (FUZ_rand(&g_rand) & ((1 << NB_TESTS_PLAYED(params))-1)) continue;
|
||||
|
||||
/* test */
|
||||
NB_TESTS_PLAYED(params)++;
|
||||
BMK_benchParam(&candidate, origBuff, benchedSize, ctx, dctx, params);
|
||||
|
||||
/* improvement found => new winner */
|
||||
if ( (candidate.cSpeed > targetSpeed)
|
||||
& ( (candidate.cSize < winner.result.cSize)
|
||||
| ((candidate.cSize == winner.result.cSize) & (candidate.cSpeed > winner.result.cSpeed)) ) )
|
||||
{
|
||||
winner.params = params;
|
||||
winner.result = candidate;
|
||||
BMK_printWinner(stdout, CUSTOM_LEVEL, winner.result, winner.params, benchedSize);
|
||||
BMK_translateAdvancedParams(winner.params);
|
||||
}
|
||||
} while (BMK_timeSpan(grillStart) < g_grillDuration_s);
|
||||
}
|
||||
/* end summary */
|
||||
|
||||
BMK_printWinner(stdout, CUSTOM_LEVEL, winner.result, winner.params, benchedSize);
|
||||
BMK_translateAdvancedParams(winner.params);
|
||||
DISPLAY("grillParams size - optimizer completed \n");
|
||||
|
||||
/* clean up*/
|
||||
ZSTD_freeCCtx(ctx);
|
||||
ZSTD_freeDCtx(dctx);
|
||||
winnerInfo = bestFeasible1;
|
||||
}
|
||||
|
||||
free(origBuff);
|
||||
return 0;
|
||||
return winnerInfo;
|
||||
}
|
||||
|
||||
/* Optimizes for a fixed strategy */
|
||||
|
||||
/* flexible parameters: iterations of failed climbing (or if we do non-random, maybe this is when everything is close to visitied)
|
||||
weight more on visit for bad results, less on good results/more on later results / ones with more failures.
|
||||
allocate memoTable here.
|
||||
*/
|
||||
static winnerInfo_t optimizeFixedStrategy(
|
||||
const buffers_t buf, const contexts_t ctx,
|
||||
const constraint_t target, paramValues_t paramTarget,
|
||||
const ZSTD_strategy strat,
|
||||
memoTable_t* memoTableArray, const int tries) {
|
||||
int i = 0;
|
||||
|
||||
paramValues_t init;
|
||||
winnerInfo_t winnerInfo, candidateInfo;
|
||||
winnerInfo = initWinnerInfo(emptyParams());
|
||||
/* so climb is given the right fixed strategy */
|
||||
paramTarget.vals[strt_ind] = strat;
|
||||
/* to pass ZSTD_checkCParams */
|
||||
paramTarget = cParamUnsetMin(paramTarget);
|
||||
|
||||
init = paramTarget;
|
||||
|
||||
for(i = 0; i < tries; i++) {
|
||||
DEBUGOUTPUT("Restart\n");
|
||||
do { randomConstrainedParams(&init, memoTableArray, strat); } while(redundantParams(init, target, buf.maxBlockSize));
|
||||
candidateInfo = climbOnce(target, memoTableArray, buf, ctx, init);
|
||||
if(compareResultLT(winnerInfo.result, candidateInfo.result, target, buf.srcSize)) {
|
||||
winnerInfo = candidateInfo;
|
||||
BMK_printWinnerOpt(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, target, buf.srcSize);
|
||||
i = 0;
|
||||
continue;
|
||||
}
|
||||
CHECKTIME(winnerInfo);
|
||||
i++;
|
||||
}
|
||||
return winnerInfo;
|
||||
}
|
||||
|
||||
/* goes best, best-1, best+1, best-2, ... */
|
||||
/* return 0 if nothing remaining */
|
||||
static int nextStrategy(const int currentStrategy, const int bestStrategy) {
|
||||
if(bestStrategy <= currentStrategy) {
|
||||
int candidate = 2 * bestStrategy - currentStrategy - 1;
|
||||
if(candidate < 1) {
|
||||
candidate = currentStrategy + 1;
|
||||
if(candidate > (int)ZSTD_btultra) {
|
||||
return 0;
|
||||
} else {
|
||||
return candidate;
|
||||
}
|
||||
} else {
|
||||
return candidate;
|
||||
}
|
||||
} else { /* bestStrategy >= currentStrategy */
|
||||
int candidate = 2 * bestStrategy - currentStrategy;
|
||||
if(candidate > (int)ZSTD_btultra) {
|
||||
candidate = currentStrategy - 1;
|
||||
if(candidate < 1) {
|
||||
return 0;
|
||||
} else {
|
||||
return candidate;
|
||||
}
|
||||
} else {
|
||||
return candidate;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/* experiment with playing with this and decay value */
|
||||
|
||||
/* main fn called when using --optimize */
|
||||
/* Does strategy selection by benchmarking default compression levels
|
||||
* then optimizes by strategy, starting with the best one and moving
|
||||
* progressively moving further away by number
|
||||
* args:
|
||||
* fileNamesTable - list of files to benchmark
|
||||
* nbFiles - length of fileNamesTable
|
||||
* dictFileName - name of dictionary file if one, else NULL
|
||||
* target - performance constraints (cSpeed, dSpeed, cMem)
|
||||
* paramTarget - parameter constraints (i.e. restriction search space to where strategy = ZSTD_fast)
|
||||
* cLevel - compression level to exceed (all solutions must be > lvl in cSpeed + ratio)
|
||||
*/
|
||||
|
||||
static int g_maxTries = 5;
|
||||
#define TRY_DECAY 1
|
||||
|
||||
static int optimizeForSize(const char* const * const fileNamesTable, const size_t nbFiles, const char* dictFileName, constraint_t target, paramValues_t paramTarget,
|
||||
const int cLevelOpt, const int cLevelRun, const U32 memoTableLog)
|
||||
{
|
||||
varInds_t varArray [NUM_PARAMS];
|
||||
int ret = 0;
|
||||
const size_t varLen = variableParams(paramTarget, varArray, dictFileName != NULL);
|
||||
winnerInfo_t winner = initWinnerInfo(emptyParams());
|
||||
memoTable_t* allMT = NULL;
|
||||
paramValues_t paramBase;
|
||||
contexts_t ctx;
|
||||
buffers_t buf;
|
||||
g_time = UTIL_getTime();
|
||||
|
||||
if(createBuffers(&buf, fileNamesTable, nbFiles)) {
|
||||
DISPLAY("unable to load files\n");
|
||||
return 1;
|
||||
}
|
||||
|
||||
if(createContexts(&ctx, dictFileName)) {
|
||||
DISPLAY("unable to load dictionary\n");
|
||||
freeBuffers(buf);
|
||||
return 2;
|
||||
}
|
||||
|
||||
if(nbFiles == 1) {
|
||||
DISPLAYLEVEL(2, "Loading %s... \r", fileNamesTable[0]);
|
||||
} else {
|
||||
DISPLAYLEVEL(2, "Loading %lu Files... \r", (unsigned long)nbFiles);
|
||||
}
|
||||
|
||||
/* sanitize paramTarget */
|
||||
optimizerAdjustInput(¶mTarget, buf.maxBlockSize);
|
||||
paramBase = cParamUnsetMin(paramTarget);
|
||||
|
||||
allMT = createMemoTableArray(paramTarget, varArray, varLen, memoTableLog);
|
||||
|
||||
if(!allMT) {
|
||||
DISPLAY("MemoTable Init Error\n");
|
||||
ret = 2;
|
||||
goto _cleanUp;
|
||||
}
|
||||
|
||||
/* default strictnesses */
|
||||
if(g_strictness == PARAM_UNSET) {
|
||||
if(g_optmode) {
|
||||
g_strictness = 100;
|
||||
} else {
|
||||
g_strictness = 90;
|
||||
}
|
||||
} else {
|
||||
if(0 >= g_strictness || g_strictness > 100) {
|
||||
DISPLAY("Strictness Outside of Bounds\n");
|
||||
ret = 4;
|
||||
goto _cleanUp;
|
||||
}
|
||||
}
|
||||
|
||||
/* use level'ing mode instead of normal target mode */
|
||||
if(g_optmode) {
|
||||
winner.params = cParamsToPVals(ZSTD_getCParams(cLevelOpt, buf.maxBlockSize, ctx.dictSize));
|
||||
if(BMK_benchParam(&winner.result, buf, ctx, winner.params)) {
|
||||
ret = 3;
|
||||
goto _cleanUp;
|
||||
}
|
||||
|
||||
g_lvltarget = winner.result;
|
||||
g_lvltarget.cSpeed *= ((double)g_strictness) / 100;
|
||||
g_lvltarget.dSpeed *= ((double)g_strictness) / 100;
|
||||
g_lvltarget.cSize /= ((double)g_strictness) / 100;
|
||||
|
||||
target.cSpeed = (U32)g_lvltarget.cSpeed;
|
||||
target.dSpeed = (U32)g_lvltarget.dSpeed;
|
||||
|
||||
BMK_printWinnerOpt(stdout, cLevelOpt, winner.result, winner.params, target, buf.srcSize);
|
||||
}
|
||||
|
||||
/* Don't want it to return anything worse than the best known result */
|
||||
if(g_singleRun) {
|
||||
BMK_result_t res;
|
||||
g_params = adjustParams(overwriteParams(cParamsToPVals(ZSTD_getCParams(cLevelRun, buf.maxBlockSize, ctx.dictSize)), g_params), buf.maxBlockSize, ctx.dictSize);
|
||||
if(BMK_benchParam(&res, buf, ctx, g_params)) {
|
||||
ret = 45;
|
||||
goto _cleanUp;
|
||||
}
|
||||
if(compareResultLT(winner.result, res, relaxTarget(target), buf.srcSize)) {
|
||||
winner.result = res;
|
||||
winner.params = g_params;
|
||||
}
|
||||
}
|
||||
|
||||
/* bench */
|
||||
DISPLAYLEVEL(2, "\r%79s\r", "");
|
||||
if(nbFiles == 1) {
|
||||
DISPLAYLEVEL(2, "optimizing for %s", fileNamesTable[0]);
|
||||
} else {
|
||||
DISPLAYLEVEL(2, "optimizing for %lu Files", (unsigned long)nbFiles);
|
||||
}
|
||||
|
||||
if(target.cSpeed != 0) { DISPLAYLEVEL(2," - limit compression speed %u MB/s", target.cSpeed >> 20); }
|
||||
if(target.dSpeed != 0) { DISPLAYLEVEL(2, " - limit decompression speed %u MB/s", target.dSpeed >> 20); }
|
||||
if(target.cMem != (U32)-1) { DISPLAYLEVEL(2, " - limit memory %u MB", target.cMem >> 20); }
|
||||
|
||||
DISPLAYLEVEL(2, "\n");
|
||||
findClockGranularity();
|
||||
|
||||
{
|
||||
paramValues_t CParams;
|
||||
|
||||
/* find best solution from default params */
|
||||
{
|
||||
/* strategy selection */
|
||||
const int maxSeeds = g_noSeed ? 1 : ZSTD_maxCLevel();
|
||||
DEBUGOUTPUT("Strategy Selection\n");
|
||||
if(paramTarget.vals[strt_ind] == PARAM_UNSET) {
|
||||
BMK_result_t candidate;
|
||||
int i;
|
||||
for (i=1; i<=maxSeeds; i++) {
|
||||
int ec;
|
||||
CParams = overwriteParams(cParamsToPVals(ZSTD_getCParams(i, buf.maxBlockSize, ctx.dictSize)), paramTarget);
|
||||
ec = BMK_benchParam(&candidate, buf, ctx, CParams);
|
||||
BMK_printWinnerOpt(stdout, i, candidate, CParams, target, buf.srcSize);
|
||||
|
||||
if(!ec && compareResultLT(winner.result, candidate, relaxTarget(target), buf.srcSize)) {
|
||||
winner.result = candidate;
|
||||
winner.params = CParams;
|
||||
}
|
||||
|
||||
CHECKTIMEGT(ret, 0, _displayCleanUp); /* if pass time limit, stop */
|
||||
/* if the current params are too slow, just stop. */
|
||||
if(target.cSpeed > candidate.cSpeed * 3 / 2) { break; }
|
||||
}
|
||||
|
||||
BMK_printWinnerOpt(stdout, CUSTOM_LEVEL, winner.result, winner.params, target, buf.srcSize);
|
||||
}
|
||||
}
|
||||
|
||||
DEBUGOUTPUT("Real Opt\n");
|
||||
/* start 'real' optimization */
|
||||
{
|
||||
int bestStrategy = (int)winner.params.vals[strt_ind];
|
||||
if(paramTarget.vals[strt_ind] == PARAM_UNSET) {
|
||||
int st = bestStrategy;
|
||||
int tries = g_maxTries;
|
||||
|
||||
{
|
||||
/* one iterations of hill climbing with the level-defined parameters. */
|
||||
winnerInfo_t w1 = climbOnce(target, allMT, buf, ctx, winner.params);
|
||||
if(compareResultLT(winner.result, w1.result, target, buf.srcSize)) {
|
||||
winner = w1;
|
||||
}
|
||||
CHECKTIMEGT(ret, 0, _displayCleanUp);
|
||||
}
|
||||
|
||||
while(st && tries > 0) {
|
||||
winnerInfo_t wc;
|
||||
DEBUGOUTPUT("StrategySwitch: %s\n", g_stratName[st]);
|
||||
|
||||
wc = optimizeFixedStrategy(buf, ctx, target, paramBase, st, allMT, tries);
|
||||
|
||||
if(compareResultLT(winner.result, wc.result, target, buf.srcSize)) {
|
||||
winner = wc;
|
||||
tries = g_maxTries;
|
||||
bestStrategy = st;
|
||||
} else {
|
||||
st = nextStrategy(st, bestStrategy);
|
||||
tries -= TRY_DECAY;
|
||||
}
|
||||
CHECKTIMEGT(ret, 0, _displayCleanUp);
|
||||
}
|
||||
} else {
|
||||
winner = optimizeFixedStrategy(buf, ctx, target, paramBase, paramTarget.vals[strt_ind], allMT, g_maxTries);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
/* no solution found */
|
||||
if(winner.result.cSize == (size_t)-1) {
|
||||
ret = 1;
|
||||
DISPLAY("No feasible solution found\n");
|
||||
goto _cleanUp;
|
||||
}
|
||||
/* end summary */
|
||||
_displayCleanUp:
|
||||
if(g_displayLevel >= 0) { BMK_displayOneResult(stdout, winner, buf.srcSize); }
|
||||
BMK_translateAdvancedParams(stdout, winner.params);
|
||||
DISPLAYLEVEL(1, "grillParams size - optimizer completed \n");
|
||||
|
||||
}
|
||||
_cleanUp:
|
||||
freeContexts(ctx);
|
||||
freeBuffers(buf);
|
||||
freeMemoTableArray(allMT);
|
||||
return ret;
|
||||
}
|
||||
|
||||
/*-************************************
|
||||
* CLI parsing functions
|
||||
**************************************/
|
||||
|
||||
/** longCommandWArg() :
|
||||
* check if *stringPtr is the same as longCommand.
|
||||
* If yes, @return 1 and advances *stringPtr to the position which immediately follows longCommand.
|
||||
* @return 0 and doesn't modify *stringPtr otherwise.
|
||||
* from zstdcli.c
|
||||
*/
|
||||
static unsigned longCommandWArg(const char** stringPtr, const char* longCommand)
|
||||
{
|
||||
size_t const comSize = strlen(longCommand);
|
||||
int const result = !strncmp(*stringPtr, longCommand, comSize);
|
||||
if (result) *stringPtr += comSize;
|
||||
return result;
|
||||
}
|
||||
|
||||
static void errorOut(const char* msg)
|
||||
@@ -772,7 +2393,9 @@ static void errorOut(const char* msg)
|
||||
static unsigned readU32FromChar(const char** stringPtr)
|
||||
{
|
||||
const char errorMsg[] = "error: numeric value too large";
|
||||
unsigned sign = 1;
|
||||
unsigned result = 0;
|
||||
if(**stringPtr == '-') { sign = (unsigned)-1; (*stringPtr)++; }
|
||||
while ((**stringPtr >='0') && (**stringPtr <='9')) {
|
||||
unsigned const max = (((unsigned)(-1)) / 10) - 1;
|
||||
if (result > max) errorOut(errorMsg);
|
||||
@@ -790,6 +2413,22 @@ static unsigned readU32FromChar(const char** stringPtr)
|
||||
if (**stringPtr=='i') (*stringPtr)++;
|
||||
if (**stringPtr=='B') (*stringPtr)++;
|
||||
}
|
||||
return result * sign;
|
||||
}
|
||||
|
||||
static double readDoubleFromChar(const char** stringPtr)
|
||||
{
|
||||
double result = 0, divide = 10;
|
||||
while ((**stringPtr >='0') && (**stringPtr <='9')) {
|
||||
result *= 10, result += **stringPtr - '0', (*stringPtr)++ ;
|
||||
}
|
||||
if(**stringPtr!='.') {
|
||||
return result;
|
||||
}
|
||||
(*stringPtr)++;
|
||||
while ((**stringPtr >='0') && (**stringPtr <='9')) {
|
||||
result += (double)(**stringPtr - '0') / divide, divide *= 10, (*stringPtr)++ ;
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
@@ -806,15 +2445,16 @@ static int usage(const char* exename)
|
||||
static int usage_advanced(void)
|
||||
{
|
||||
DISPLAY( "\nAdvanced options :\n");
|
||||
DISPLAY( " -T# : set level 1 speed objective \n");
|
||||
DISPLAY( " -B# : cut input into blocks of size # (default : single block) \n");
|
||||
DISPLAY( " -i# : iteration loops [1-9](default : %i) \n", NBLOOPS);
|
||||
DISPLAY( " -O# : find Optimized parameters for # MB/s compression speed (default : 0) \n");
|
||||
DISPLAY( " -S : Single run \n");
|
||||
DISPLAY( " --zstd : Single run, parameter selection same as zstdcli \n");
|
||||
DISPLAY( " -P# : generated sample compressibility (default : %.1f%%) \n", COMPRESSIBILITY_DEFAULT * 100);
|
||||
DISPLAY( " -t# : Caps runtime of operation in seconds (default : %u seconds (%.1f hours)) \n", (U32)g_grillDuration_s, g_grillDuration_s / 3600);
|
||||
DISPLAY( " -v : Prints Benchmarking output\n");
|
||||
DISPLAY( " -T# : set level 1 speed objective \n");
|
||||
DISPLAY( " -B# : cut input into blocks of size # (default : single block) \n");
|
||||
DISPLAY( " --optimize= : same as -O with more verbose syntax (see README.md)\n");
|
||||
DISPLAY( " -S : Single run \n");
|
||||
DISPLAY( " --zstd : Single run, parameter selection same as zstdcli \n");
|
||||
DISPLAY( " -P# : generated sample compressibility (default : %.1f%%) \n", COMPRESSIBILITY_DEFAULT * 100);
|
||||
DISPLAY( " -t# : Caps runtime of operation in seconds (default : %u seconds (%.1f hours)) \n", g_timeLimit_s, (double)g_timeLimit_s / 3600);
|
||||
DISPLAY( " -v : Prints Benchmarking output\n");
|
||||
DISPLAY( " -D : Next argument dictionary file\n");
|
||||
DISPLAY( " -s : Seperate Files\n");
|
||||
return 0;
|
||||
}
|
||||
|
||||
@@ -825,41 +2465,87 @@ static int badusage(const char* exename)
|
||||
return 1;
|
||||
}
|
||||
|
||||
#define PARSE_SUB_ARGS(stringLong, stringShort, variable) { if (longCommandWArg(&argument, stringLong) || longCommandWArg(&argument, stringShort)) { variable = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; } }
|
||||
/* 1 if successful parse, 0 otherwise */
|
||||
static int parse_params(const char** argptr, paramValues_t* pv) {
|
||||
int matched = 0;
|
||||
const char* argOrig = *argptr;
|
||||
varInds_t v;
|
||||
for(v = 0; v < NUM_PARAMS; v++) {
|
||||
if(longCommandWArg(argptr,g_shortParamNames[v]) || longCommandWArg(argptr, g_paramNames[v])) {
|
||||
if(**argptr == '=') {
|
||||
(*argptr)++;
|
||||
pv->vals[v] = readU32FromChar(argptr);
|
||||
matched = 1;
|
||||
break;
|
||||
}
|
||||
}
|
||||
/* reset and try again */
|
||||
*argptr = argOrig;
|
||||
}
|
||||
return matched;
|
||||
}
|
||||
|
||||
/*-************************************
|
||||
* Main
|
||||
**************************************/
|
||||
|
||||
int main(int argc, const char** argv)
|
||||
{
|
||||
int i,
|
||||
filenamesStart=0,
|
||||
result;
|
||||
const char* exename=argv[0];
|
||||
const char* input_filename=0;
|
||||
U32 optimizer = 0;
|
||||
const char* input_filename = NULL;
|
||||
const char* dictFileName = NULL;
|
||||
U32 main_pause = 0;
|
||||
U32 targetSpeed = 0;
|
||||
int cLevelOpt = 0, cLevelRun = 0;
|
||||
int seperateFiles = 0;
|
||||
double compressibility = COMPRESSIBILITY_DEFAULT;
|
||||
U32 memoTableLog = PARAM_UNSET;
|
||||
constraint_t target = { 0, 0, (U32)-1 };
|
||||
|
||||
paramValues_t paramTarget = emptyParams();
|
||||
g_params = emptyParams();
|
||||
|
||||
assert(argc>=1); /* for exename */
|
||||
|
||||
/* Welcome message */
|
||||
DISPLAY(WELCOME_MESSAGE);
|
||||
|
||||
for(i=1; i<argc; i++) {
|
||||
const char* argument = argv[i];
|
||||
DEBUGOUTPUT("%d: %s\n", i, argument);
|
||||
assert(argument != NULL);
|
||||
|
||||
if(!strcmp(argument,"--no-seed")) { g_noSeed = 1; continue; }
|
||||
|
||||
/* Decode command (note : aggregated commands are allowed) */
|
||||
if (longCommandWArg(&argument, "--zstd=")) {
|
||||
g_singleRun = 1;
|
||||
g_params = ZSTD_getCParams(2, g_blockSize, 0);
|
||||
if (longCommandWArg(&argument, "--optimize=")) {
|
||||
g_optimizer = 1;
|
||||
for ( ; ;) {
|
||||
if (longCommandWArg(&argument, "windowLog=") || longCommandWArg(&argument, "wlog=")) { g_params.windowLog = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; }
|
||||
if (longCommandWArg(&argument, "chainLog=") || longCommandWArg(&argument, "clog=")) { g_params.chainLog = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; }
|
||||
if (longCommandWArg(&argument, "hashLog=") || longCommandWArg(&argument, "hlog=")) { g_params.hashLog = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; }
|
||||
if (longCommandWArg(&argument, "searchLog=") || longCommandWArg(&argument, "slog=")) { g_params.searchLog = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; }
|
||||
if (longCommandWArg(&argument, "searchLength=") || longCommandWArg(&argument, "slen=")) { g_params.searchLength = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; }
|
||||
if (longCommandWArg(&argument, "targetLength=") || longCommandWArg(&argument, "tlen=")) { g_params.targetLength = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; }
|
||||
if (longCommandWArg(&argument, "strategy=") || longCommandWArg(&argument, "strat=")) { g_params.strategy = (ZSTD_strategy)(readU32FromChar(&argument)); if (argument[0]==',') { argument++; continue; } else break; }
|
||||
if (longCommandWArg(&argument, "level=") || longCommandWArg(&argument, "lvl=")) { g_params = ZSTD_getCParams(readU32FromChar(&argument), g_blockSize, 0); if (argument[0]==',') { argument++; continue; } else break; }
|
||||
if(parse_params(&argument, ¶mTarget)) { if(argument[0] == ',') { argument++; continue; } else break; }
|
||||
PARSE_SUB_ARGS("compressionSpeed=" , "cSpeed=", target.cSpeed);
|
||||
PARSE_SUB_ARGS("decompressionSpeed=", "dSpeed=", target.dSpeed);
|
||||
PARSE_SUB_ARGS("compressionMemory=" , "cMem=", target.cMem);
|
||||
PARSE_SUB_ARGS("strict=", "stc=", g_strictness);
|
||||
PARSE_SUB_ARGS("maxTries=", "tries=", g_maxTries);
|
||||
PARSE_SUB_ARGS("memoLimitLog=", "memLog=", memoTableLog);
|
||||
if (longCommandWArg(&argument, "level=") || longCommandWArg(&argument, "lvl=")) { cLevelOpt = readU32FromChar(&argument); g_optmode = 1; if (argument[0]==',') { argument++; continue; } else break; }
|
||||
if (longCommandWArg(&argument, "speedForRatio=") || longCommandWArg(&argument, "speedRatio=")) { g_ratioMultiplier = readDoubleFromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; }
|
||||
|
||||
DISPLAY("invalid optimization parameter \n");
|
||||
return 1;
|
||||
}
|
||||
|
||||
if (argument[0] != 0) {
|
||||
DISPLAY("invalid --optimize= format\n");
|
||||
return 1; /* check the end of string */
|
||||
}
|
||||
continue;
|
||||
} else if (longCommandWArg(&argument, "--zstd=")) {
|
||||
/* Decode command (note : aggregated commands are allowed) */
|
||||
g_singleRun = 1;
|
||||
for ( ; ;) {
|
||||
if(parse_params(&argument, &g_params)) { if(argument[0] == ',') { argument++; continue; } else break; }
|
||||
if (longCommandWArg(&argument, "level=") || longCommandWArg(&argument, "lvl=")) { cLevelRun = readU32FromChar(&argument); g_params = emptyParams(); if (argument[0]==',') { argument++; continue; } else break; }
|
||||
|
||||
DISPLAY("invalid compression parameter \n");
|
||||
return 1;
|
||||
}
|
||||
@@ -868,7 +2554,45 @@ int main(int argc, const char** argv)
|
||||
DISPLAY("invalid --zstd= format\n");
|
||||
return 1; /* check the end of string */
|
||||
}
|
||||
continue;
|
||||
/* if not return, success */
|
||||
|
||||
} else if (longCommandWArg(&argument, "--display=")) {
|
||||
/* Decode command (note : aggregated commands are allowed) */
|
||||
memset(g_silenceParams, 1, sizeof(g_silenceParams));
|
||||
for ( ; ;) {
|
||||
int found = 0;
|
||||
varInds_t v;
|
||||
for(v = 0; v < NUM_PARAMS; v++) {
|
||||
if(longCommandWArg(&argument, g_shortParamNames[v]) || longCommandWArg(&argument, g_paramNames[v])) {
|
||||
g_silenceParams[v] = 0;
|
||||
found = 1;
|
||||
}
|
||||
}
|
||||
if(longCommandWArg(&argument, "compressionParameters") || longCommandWArg(&argument, "cParams")) {
|
||||
for(v = 0; v <= strt_ind; v++) {
|
||||
g_silenceParams[v] = 0;
|
||||
}
|
||||
found = 1;
|
||||
}
|
||||
|
||||
|
||||
if(found) {
|
||||
if(argument[0]==',') {
|
||||
continue;
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
}
|
||||
DISPLAY("invalid parameter name parameter \n");
|
||||
return 1;
|
||||
}
|
||||
|
||||
if (argument[0] != 0) {
|
||||
DISPLAY("invalid --display format\n");
|
||||
return 1; /* check the end of string */
|
||||
}
|
||||
continue;
|
||||
} else if (argument[0]=='-') {
|
||||
argument++;
|
||||
|
||||
@@ -882,66 +2606,58 @@ int main(int argc, const char** argv)
|
||||
|
||||
/* Pause at the end (hidden option) */
|
||||
case 'p': main_pause = 1; argument++; break;
|
||||
/* Modify Nb Iterations */
|
||||
|
||||
case 'i':
|
||||
argument++;
|
||||
g_nbIterations = readU32FromChar(&argument);
|
||||
break;
|
||||
|
||||
/* Sample compressibility (when no file provided) */
|
||||
case 'P':
|
||||
argument++;
|
||||
{ U32 const proba32 = readU32FromChar(&argument);
|
||||
g_compressibility = (double)proba32 / 100.;
|
||||
compressibility = (double)proba32 / 100.;
|
||||
}
|
||||
break;
|
||||
|
||||
case 'O':
|
||||
argument++;
|
||||
optimizer = 1;
|
||||
targetSpeed = readU32FromChar(&argument);
|
||||
break;
|
||||
|
||||
/* Run Single conf */
|
||||
case 'S':
|
||||
g_singleRun = 1;
|
||||
argument++;
|
||||
g_params = ZSTD_getCParams(2, g_blockSize, 0);
|
||||
for ( ; ; ) {
|
||||
switch(*argument)
|
||||
{
|
||||
case 'w':
|
||||
argument++;
|
||||
g_params.windowLog = readU32FromChar(&argument);
|
||||
g_params.vals[wlog_ind] = readU32FromChar(&argument);
|
||||
continue;
|
||||
case 'c':
|
||||
argument++;
|
||||
g_params.chainLog = readU32FromChar(&argument);
|
||||
g_params.vals[clog_ind] = readU32FromChar(&argument);
|
||||
continue;
|
||||
case 'h':
|
||||
argument++;
|
||||
g_params.hashLog = readU32FromChar(&argument);
|
||||
g_params.vals[hlog_ind] = readU32FromChar(&argument);
|
||||
continue;
|
||||
case 's':
|
||||
argument++;
|
||||
g_params.searchLog = readU32FromChar(&argument);
|
||||
g_params.vals[slog_ind] = readU32FromChar(&argument);
|
||||
continue;
|
||||
case 'l': /* search length */
|
||||
argument++;
|
||||
g_params.searchLength = readU32FromChar(&argument);
|
||||
g_params.vals[slen_ind] = readU32FromChar(&argument);
|
||||
continue;
|
||||
case 't': /* target length */
|
||||
argument++;
|
||||
g_params.targetLength = readU32FromChar(&argument);
|
||||
g_params.vals[tlen_ind] = readU32FromChar(&argument);
|
||||
continue;
|
||||
case 'S': /* strategy */
|
||||
argument++;
|
||||
g_params.strategy = (ZSTD_strategy)readU32FromChar(&argument);
|
||||
g_params.vals[strt_ind] = readU32FromChar(&argument);
|
||||
continue;
|
||||
case 'f': /* forceAttachDict */
|
||||
argument++;
|
||||
g_params.vals[fadt_ind] = readU32FromChar(&argument);
|
||||
continue;
|
||||
case 'L':
|
||||
{ int const cLevel = readU32FromChar(&argument);
|
||||
g_params = ZSTD_getCParams(cLevel, g_blockSize, 0);
|
||||
{ argument++;
|
||||
cLevelRun = readU32FromChar(&argument);
|
||||
g_params = emptyParams();
|
||||
continue;
|
||||
}
|
||||
default : ;
|
||||
@@ -967,7 +2683,32 @@ int main(int argc, const char** argv)
|
||||
/* caps runtime (in seconds) */
|
||||
case 't':
|
||||
argument++;
|
||||
g_grillDuration_s = (double)readU32FromChar(&argument);
|
||||
g_timeLimit_s = readU32FromChar(&argument);
|
||||
break;
|
||||
|
||||
case 's':
|
||||
argument++;
|
||||
seperateFiles = 1;
|
||||
break;
|
||||
|
||||
case 'q':
|
||||
while (argument[0] == 'q') { argument++; g_displayLevel--; }
|
||||
break;
|
||||
|
||||
case 'v':
|
||||
while (argument[0] == 'v') { argument++; g_displayLevel++; }
|
||||
break;
|
||||
|
||||
/* load dictionary file (only applicable for optimizer rn) */
|
||||
case 'D':
|
||||
if(i == argc - 1) { /* last argument, return error. */
|
||||
DISPLAY("Dictionary file expected but not given : %d\n", i);
|
||||
return 1;
|
||||
} else {
|
||||
i++;
|
||||
dictFileName = argv[i];
|
||||
argument += strlen(argument);
|
||||
}
|
||||
break;
|
||||
|
||||
/* Unknown command */
|
||||
@@ -981,19 +2722,35 @@ int main(int argc, const char** argv)
|
||||
if (!input_filename) { input_filename=argument; filenamesStart=i; continue; }
|
||||
}
|
||||
|
||||
/* Welcome message */
|
||||
DISPLAYLEVEL(2, WELCOME_MESSAGE);
|
||||
|
||||
if (filenamesStart==0) {
|
||||
if (optimizer) {
|
||||
if (g_optimizer) {
|
||||
DISPLAY("Optimizer Expects File\n");
|
||||
return 1;
|
||||
} else {
|
||||
result = benchSample();
|
||||
result = benchSample(compressibility, cLevelRun);
|
||||
}
|
||||
} else {
|
||||
if (optimizer) {
|
||||
result = optimizeForSize(input_filename, targetSpeed);
|
||||
if(seperateFiles) {
|
||||
for(i = 0; i < argc - filenamesStart; i++) {
|
||||
if (g_optimizer) {
|
||||
result = optimizeForSize(argv+filenamesStart + i, 1, dictFileName, target, paramTarget, cLevelOpt, cLevelRun, memoTableLog);
|
||||
if(result) { DISPLAY("Error on File %d", i); return result; }
|
||||
} else {
|
||||
result = benchFiles(argv+filenamesStart + i, 1, dictFileName, cLevelRun);
|
||||
if(result) { DISPLAY("Error on File %d", i); return result; }
|
||||
}
|
||||
}
|
||||
} else {
|
||||
result = benchFiles(argv+filenamesStart, argc-filenamesStart);
|
||||
} }
|
||||
if (g_optimizer) {
|
||||
result = optimizeForSize(argv+filenamesStart, argc-filenamesStart, dictFileName, target, paramTarget, cLevelOpt, cLevelRun, memoTableLog);
|
||||
} else {
|
||||
result = benchFiles(argv+filenamesStart, argc-filenamesStart, dictFileName, cLevelRun);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (main_pause) { int unused; printf("press enter...\n"); unused = getchar(); (void)unused; }
|
||||
|
||||
|
||||
Reference in New Issue
Block a user