ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #1168 from GeorgeLu97/paramgrillfeatures

Browse Source 
Have paramgrill share bench.c benchmarking function
This commit is contained in:
Yann Collet
2018-06-13 11:38:29 -04:00
committed by GitHub
parent 750ee87a92 01d940b670
commit c986dbf241
6 changed files with 210 additions and 259 deletions
Download Patch File Download Diff File Expand all files Collapse all files
programs/bench.c
+104 -53
View File
@@ -41,6 +41,7 @@
#include "zstd.h"
#include "datagen.h" /* RDG_genBuffer */
#include "xxhash.h"
#include "bench.h"
/* *************************************
@@ -70,16 +71,16 @@ static U32 g_compressibilityDefault = 50;
* console display
***************************************/
#define DISPLAY(...) fprintf(stderr, __VA_ARGS__)
#define DISPLAYLEVEL(l, ...) if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); }
static int g_displayLevel = 2; /* 0 : no display; 1: errors; 2 : + result + interaction + warnings; 3 : + progression; 4 : + information */
#define DISPLAYLEVEL(l, ...) if (displayLevel>=l) { DISPLAY(__VA_ARGS__); }
/* 0 : no display; 1: errors; 2 : + result + interaction + warnings; 3 : + progression; 4 : + information */
static const U64 g_refreshRate = SEC_TO_MICRO / 6;
static UTIL_time_t g_displayClock = UTIL_TIME_INITIALIZER;
#define DISPLAYUPDATE(l, ...) { if (g_displayLevel>=l) { \
if ((UTIL_clockSpanMicro(g_displayClock) > g_refreshRate) || (g_displayLevel>=4)) \
#define DISPLAYUPDATE(l, ...) { if (displayLevel>=l) { \
if ((UTIL_clockSpanMicro(g_displayClock) > g_refreshRate) || (displayLevel>=4)) \
{ g_displayClock = UTIL_getTime(); DISPLAY(__VA_ARGS__); \
if (g_displayLevel>=4) fflush(stderr); } } }
if (displayLevel>=4) fflush(stderr); } } }
/* *************************************
@@ -104,30 +105,34 @@ static UTIL_time_t g_displayClock = UTIL_TIME_INITIALIZER;
static int g_additionalParam = 0;
static U32 g_decodeOnly = 0;
void BMK_setNotificationLevel(unsigned level) { g_displayLevel=level; }
void BMK_setAdditionalParam(int additionalParam) { g_additionalParam=additionalParam; }
//TODO : Deal with DISPLAYLEVEL for all these set functions
static U32 g_nbSeconds = BMK_TIMETEST_DEFAULT_S;
void BMK_setNbSeconds(unsigned nbSeconds)
{
g_nbSeconds = nbSeconds;
DISPLAYLEVEL(3, "- test >= %u seconds per compression / decompression - \n", g_nbSeconds);
DISPLAY("- test >= %u seconds per compression / decompression - \n", g_nbSeconds);
}
static size_t g_blockSize = 0;
void BMK_setBlockSize(size_t blockSize)
{
g_blockSize = blockSize;
if (g_blockSize) DISPLAYLEVEL(2, "using blocks of size %u KB \n", (U32)(blockSize>>10));
if (g_blockSize) DISPLAY("using blocks of size %u KB \n", (U32)(blockSize>>10));
}
void BMK_setDecodeOnlyMode(unsigned decodeFlag) { g_decodeOnly = (decodeFlag>0); }
static U32 g_nbWorkers = 0;
void BMK_setNbWorkers(unsigned nbWorkers) {
#ifndef ZSTD_MULTITHREAD
if (nbWorkers > 0) DISPLAYLEVEL(2, "Note : multi-threading is disabled \n");
if (nbWorkers > 0) DISPLAY("Note : multi-threading is disabled \n");
#endif
g_nbWorkers = nbWorkers;
}
@@ -168,7 +173,6 @@ void BMK_setLdmHashEveryLog(unsigned ldmHashEveryLog) {
g_ldmHashEveryLog = ldmHashEveryLog;
}
/* ********************************************************
* Bench functions
**********************************************************/
@@ -182,18 +186,18 @@ typedef struct {
size_t resSize;
} blockParam_t;
#undef MIN
#undef MAX
#define MIN(a,b) ((a) < (b) ? (a) : (b))
#define MAX(a,b) ((a) > (b) ? (a) : (b))
static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
const char* displayName, int cLevel,
const size_t* fileSizes, U32 nbFiles,
BMK_return_t BMK_benchMem(const void* srcBuffer, size_t srcSize,
const size_t* fileSizes, unsigned nbFiles,
const int cLevel, const ZSTD_compressionParameters* comprParams,
const void* dictBuffer, size_t dictBufferSize,
const ZSTD_compressionParameters* const comprParams)
ZSTD_CCtx* ctx, ZSTD_DCtx* dctx,
int displayLevel, const char* displayName)
{
size_t const blockSize = ((g_blockSize>=32 && !g_decodeOnly) ? g_blockSize : srcSize) + (!srcSize) /* avoid div by 0 */ ;
U32 const maxNbBlocks = (U32) ((srcSize + (blockSize-1)) / blockSize) + nbFiles;
@@ -201,17 +205,20 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
size_t const maxCompressedSize = ZSTD_compressBound(srcSize) + (maxNbBlocks * 1024); /* add some room for safety */
void* const compressedBuffer = malloc(maxCompressedSize);
void* resultBuffer = malloc(srcSize);
ZSTD_CCtx* const ctx = ZSTD_createCCtx();
ZSTD_DCtx* const dctx = ZSTD_createDCtx();
BMK_return_t results;
size_t const loadedCompressedSize = srcSize;
size_t cSize = 0;
double ratio = 0.;
U32 nbBlocks;
/* checks */
if (!compressedBuffer || !resultBuffer || !blockTable || !ctx || !dctx)
if (!compressedBuffer || !resultBuffer || !blockTable)
EXM_THROW(31, "allocation error : not enough memory");
if(!ctx || !dctx)
EXM_THROW(31, "error: passed in null context");
/* init */
if (strlen(displayName)>17) displayName += strlen(displayName)-17; /* display last 17 characters */
if (g_nbWorkers==1) g_nbWorkers=0; /* prefer synchronous mode */
@@ -369,10 +376,12 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
cSize = 0;
{ U32 blockNb; for (blockNb=0; blockNb<nbBlocks; blockNb++) cSize += blockTable[blockNb].cSize; }
ratio = (double)srcSize / (double)cSize;
results.result.cSize = cSize;
markNb = (markNb+1) % NB_MARKS;
{ int const ratioAccuracy = (ratio < 10.) ? 3 : 2;
double const compressionSpeed = ((double)srcSize / fastestC) * 1000;
int const cSpeedAccuracy = (compressionSpeed < 10.) ? 2 : 1;
results.result.cSpeed = compressionSpeed * 1000000;
DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.*f),%6.*f MB/s\r",
marks[markNb], displayName, (U32)srcSize, (U32)cSize,
ratioAccuracy, ratio,
@@ -428,6 +437,8 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
double const compressionSpeed = ((double)srcSize / fastestC) * 1000;
int const cSpeedAccuracy = (compressionSpeed < 10.) ? 2 : 1;
double const decompressionSpeed = ((double)srcSize / fastestD) * 1000;
results.result.cSpeed = compressionSpeed * 1000000;
results.result.dSpeed = decompressionSpeed * 1000000;
DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.*f),%6.*f MB/s ,%6.1f MB/s \r",
marks[markNb], displayName, (U32)srcSize, (U32)cSize,
ratioAccuracy, ratio,
@@ -475,7 +486,7 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
#endif
} /* for (testNb = 1; testNb <= (g_nbSeconds + !g_nbSeconds); testNb++) */
if (g_displayLevel == 1) { /* hidden display mode -q, used by python speed benchmark */
if (displayLevel == 1) { /* hidden display mode -q, used by python speed benchmark */
double const cSpeed = ((double)srcSize / fastestC) * 1000;
double const dSpeed = ((double)srcSize / fastestD) * 1000;
if (g_additionalParam)
@@ -490,11 +501,30 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
free(blockTable);
free(compressedBuffer);
free(resultBuffer);
ZSTD_freeCCtx(ctx);
ZSTD_freeDCtx(dctx);
return 0;
results.errorCode = 0;
return results;
}
static void BMK_benchMemCtxless(const void* srcBuffer, size_t srcSize,
const size_t* fileSizes, unsigned nbFiles,
int cLevel, const ZSTD_compressionParameters* const comprParams,
const void* dictBuffer, size_t dictBufferSize,
int displayLevel, const char* displayName)
{
ZSTD_CCtx* ctx = ZSTD_createCCtx();
ZSTD_DCtx* dctx = ZSTD_createDCtx();
if(ctx == NULL || dctx == NULL) {
EXM_THROW(12, "not enough memory for contexts");
}
BMK_benchMem(srcBuffer, srcSize,
fileSizes, nbFiles,
cLevel, comprParams,
dictBuffer, dictBufferSize,
ctx, dctx,
displayLevel, displayName);
ZSTD_freeCCtx(ctx);
ZSTD_freeDCtx(dctx);
}
static size_t BMK_findMaxMem(U64 requiredMem)
{
@@ -514,11 +544,12 @@ static size_t BMK_findMaxMem(U64 requiredMem)
return (size_t)(requiredMem);
}
/* returns average stats over all range [cLevel, cLevelLast] */
static void BMK_benchCLevel(const void* srcBuffer, size_t benchedSize,
const char* displayName, int cLevel, int cLevelLast,
const size_t* fileSizes, unsigned nbFiles,
const int cLevel, const int cLevelLast, const ZSTD_compressionParameters* comprParams,
const void* dictBuffer, size_t dictBufferSize,
const ZSTD_compressionParameters* const compressionParams)
int displayLevel, const char* displayName)
{
int l;
@@ -531,16 +562,19 @@ static void BMK_benchCLevel(const void* srcBuffer, size_t benchedSize,
SET_REALTIME_PRIORITY;
}
if (g_displayLevel == 1 && !g_additionalParam)
if (displayLevel == 1 && !g_additionalParam)
DISPLAY("bench %s %s: input %u bytes, %u seconds, %u KB blocks\n", ZSTD_VERSION_STRING, ZSTD_GIT_COMMIT_STRING, (U32)benchedSize, g_nbSeconds, (U32)(g_blockSize>>10));
for (l=cLevel; l <= cLevelLast; l++) {
if (l==0) continue; /* skip level 0 */
BMK_benchMem(srcBuffer, benchedSize,
displayName, l,
fileSizes, nbFiles,
dictBuffer, dictBufferSize, compressionParams);
BMK_benchMemCtxless(srcBuffer, benchedSize,
fileSizes, nbFiles,
l, comprParams,
dictBuffer, dictBufferSize,
displayLevel, displayName);
}
return;
}
@@ -548,8 +582,8 @@ static void BMK_benchCLevel(const void* srcBuffer, size_t benchedSize,
* Loads `buffer` with content of files listed within `fileNamesTable`.
* At most, fills `buffer` entirely. */
static void BMK_loadFiles(void* buffer, size_t bufferSize,
size_t* fileSizes,
const char* const * const fileNamesTable, unsigned nbFiles)
size_t* fileSizes, const char* const * const fileNamesTable,
unsigned nbFiles, int displayLevel)
{
size_t pos = 0, totalSize = 0;
unsigned n;
@@ -582,9 +616,8 @@ static void BMK_loadFiles(void* buffer, size_t bufferSize,
}
static void BMK_benchFileTable(const char* const * const fileNamesTable, unsigned const nbFiles,
const char* const dictFileName,
int const cLevel, int const cLevelLast,
const ZSTD_compressionParameters* const compressionParams)
const char* const dictFileName, int const cLevel, int const cLevelLast,
const ZSTD_compressionParameters* const compressionParams, int displayLevel)
{
void* srcBuffer;
size_t benchedSize;
@@ -605,7 +638,7 @@ static void BMK_benchFileTable(const char* const * const fileNamesTable, unsigne
if (dictBuffer==NULL)
EXM_THROW(11, "not enough memory for dictionary (%u bytes)",
(U32)dictBufferSize);
BMK_loadFiles(dictBuffer, dictBufferSize, fileSizes, &dictFileName, 1);
BMK_loadFiles(dictBuffer, dictBufferSize, fileSizes, &dictFileName, 1, displayLevel);
}
/* Memory allocation & restrictions */
@@ -617,28 +650,33 @@ static void BMK_benchFileTable(const char* const * const fileNamesTable, unsigne
if (!srcBuffer) EXM_THROW(12, "not enough memory");
/* Load input buffer */
BMK_loadFiles(srcBuffer, benchedSize, fileSizes, fileNamesTable, nbFiles);
BMK_loadFiles(srcBuffer, benchedSize, fileSizes, fileNamesTable, nbFiles, displayLevel);
/* Bench */
if (g_separateFiles) {
const BYTE* srcPtr = (const BYTE*)srcBuffer;
U32 fileNb;
BMK_result_t* resultarray = (BMK_result_t*)malloc(sizeof(BMK_result_t) * nbFiles);
if(resultarray == NULL) EXM_THROW(12, "not enough memory");
for (fileNb=0; fileNb<nbFiles; fileNb++) {
size_t const fileSize = fileSizes[fileNb];
BMK_benchCLevel(srcPtr, fileSize,
fileNamesTable[fileNb], cLevel, cLevelLast,
fileSizes+fileNb, 1,
dictBuffer, dictBufferSize, compressionParams);
fileSizes+fileNb, 1,
cLevel, cLevelLast, compressionParams,
dictBuffer, dictBufferSize,
displayLevel, fileNamesTable[fileNb]);
srcPtr += fileSize;
}
} else {
char mfName[20] = {0};
snprintf (mfName, sizeof(mfName), " %u files", nbFiles);
{ const char* const displayName = (nbFiles > 1) ? mfName : fileNamesTable[0];
BMK_benchCLevel(srcBuffer, benchedSize,
displayName, cLevel, cLevelLast,
fileSizes, nbFiles,
dictBuffer, dictBufferSize, compressionParams);
BMK_benchCLevel(srcBuffer, benchedSize,
fileSizes, nbFiles,
cLevel, cLevelLast, compressionParams,
dictBuffer, dictBufferSize,
displayLevel, displayName);
} }
/* clean up */
@@ -649,7 +687,8 @@ static void BMK_benchFileTable(const char* const * const fileNamesTable, unsigne
static void BMK_syntheticTest(int cLevel, int cLevelLast, double compressibility,
const ZSTD_compressionParameters* compressionParams)
const ZSTD_compressionParameters* compressionParams,
int displayLevel)
{
char name[20] = {0};
size_t benchedSize = 10000000;
@@ -663,17 +702,21 @@ static void BMK_syntheticTest(int cLevel, int cLevelLast, double compressibility
/* Bench */
snprintf (name, sizeof(name), "Synthetic %2u%%", (unsigned)(compressibility*100));
BMK_benchCLevel(srcBuffer, benchedSize, name, cLevel, cLevelLast, &benchedSize, 1, NULL, 0, compressionParams);
BMK_benchCLevel(srcBuffer, benchedSize,
&benchedSize, 1,
cLevel, cLevelLast, compressionParams,
NULL, 0,
displayLevel, name);
/* clean up */
free(srcBuffer);
}
int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles,
const char* dictFileName,
int cLevel, int cLevelLast,
const ZSTD_compressionParameters* compressionParams)
static void BMK_benchFilesFull(const char** fileNamesTable, unsigned nbFiles,
const char* dictFileName,
int cLevel, int cLevelLast,
const ZSTD_compressionParameters* compressionParams, int displayLevel)
{
double const compressibility = (double)g_compressibilityDefault / 100;
@@ -684,8 +727,16 @@ int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles,
DISPLAYLEVEL(2, "Benchmarking levels from %d to %d\n", cLevel, cLevelLast);
if (nbFiles == 0)
BMK_syntheticTest(cLevel, cLevelLast, compressibility, compressionParams);
BMK_syntheticTest(cLevel, cLevelLast, compressibility, compressionParams, displayLevel);
else
BMK_benchFileTable(fileNamesTable, nbFiles, dictFileName, cLevel, cLevelLast, compressionParams);
BMK_benchFileTable(fileNamesTable, nbFiles, dictFileName, cLevel, cLevelLast, compressionParams, displayLevel);
}
int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles,
const char* dictFileName,
int cLevel, int cLevelLast,
const ZSTD_compressionParameters* compressionParams,
int displayLevel) {
BMK_benchFilesFull(fileNamesTable, nbFiles, dictFileName, cLevel, cLevelLast, compressionParams, displayLevel);
return 0;
}
programs/bench.h
+31 -1
View File
@@ -8,6 +8,9 @@
* You may select, at your option, one of the above-listed licenses.
*/
#if defined (__cplusplus)
extern "C" {
#endif
#ifndef BENCH_H_121279284357
#define BENCH_H_121279284357
@@ -16,8 +19,31 @@
#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_compressionParameters */
#include "zstd.h" /* ZSTD_compressionParameters */
typedef struct {
size_t cSize;
double cSpeed; /* bytes / sec */
double dSpeed;
} BMK_result_t;
/* 0 = no Error */
typedef struct {
int errorCode;
BMK_result_t result;
} BMK_return_t;
/* called in cli */
int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles, const char* dictFileName,
int cLevel, int cLevelLast, const ZSTD_compressionParameters* compressionParams);
int cLevel, int cLevelLast, const ZSTD_compressionParameters* compressionParams,
int displayLevel);
/* basic benchmarking function, called in paramgrill
* ctx, dctx must be valid */
BMK_return_t BMK_benchMem(const void* srcBuffer, size_t srcSize,
const size_t* fileSizes, unsigned nbFiles,
const int cLevel, const ZSTD_compressionParameters* comprParams,
const void* dictBuffer, size_t dictBufferSize,
ZSTD_CCtx* ctx, ZSTD_DCtx* dctx,
int displayLevel, const char* displayName);
/* Set Parameters */
void BMK_setNbSeconds(unsigned nbLoops);
@@ -35,3 +61,7 @@ void BMK_setLdmBucketSizeLog(unsigned ldmBucketSizeLog);
void BMK_setLdmHashEveryLog(unsigned ldmHashEveryLog);
#endif /* BENCH_H_121279284357 */
#if defined (__cplusplus)
}
#endif
programs/zstdcli.c
+1 -2
View File
@@ -801,7 +801,6 @@ int main(int argCount, const char* argv[])
/* Check if benchmark is selected */
if (operation==zom_bench) {
#ifndef ZSTD_NOBENCH
BMK_setNotificationLevel(g_displayLevel);
BMK_setSeparateFiles(separateFiles);
BMK_setBlockSize(blockSize);
BMK_setNbWorkers(nbWorkers);
@@ -816,7 +815,7 @@ int main(int argCount, const char* argv[])
if (g_ldmHashEveryLog != LDM_PARAM_DEFAULT) {
BMK_setLdmHashEveryLog(g_ldmHashEveryLog);
}
BMK_benchFiles(filenameTable, filenameIdx, dictFileName, cLevel, cLevelLast, &compressionParams);
BMK_benchFiles(filenameTable, filenameIdx, dictFileName, cLevel, cLevelLast, &compressionParams, g_displayLevel);
#else
(void)bench_nbSeconds; (void)blockSize; (void)setRealTimePrio; (void)separateFiles;
#endif