ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

convert benchmark unit to use the new naming scheme

Browse Source 
chunks instead of blocks
This commit is contained in:
Yann Collet
2025-03-04 13:04:59 -08:00
parent d5dbdd6ece
commit fcfb3160dc
3 changed files with 53 additions and 64 deletions
Download Patch File Download Diff File Expand all files Collapse all files
programs/benchzstd.c
+51 -62
View File
@@ -208,7 +208,7 @@ BMK_advancedParams_t BMK_initAdvancedParams(void)
BMK_advancedParams_t const res = { BMK_advancedParams_t const res = {
BMK_both, /* mode */ BMK_both, /* mode */
BMK_TIMETEST_DEFAULT_S, /* nbSeconds */ BMK_TIMETEST_DEFAULT_S, /* nbSeconds */
0, /* blockSize */ 0, /* chunkSizeMax */
0, /* targetCBlockSize */ 0, /* targetCBlockSize */
0, /* nbWorkers */ 0, /* nbWorkers */
0, /* realTime */ 0, /* realTime */
@@ -227,16 +227,6 @@ BMK_advancedParams_t BMK_initAdvancedParams(void)
/* ******************************************************** /* ********************************************************
* Bench functions * Bench functions
**********************************************************/ **********************************************************/
typedef struct {
const void* srcPtr;
size_t srcSize;
void* cPtr;
size_t cRoom;
size_t cSize;
void* resPtr;
size_t resSize;
} blockParam_t;
#undef MIN #undef MIN
#undef MAX #undef MAX
#define MIN(a, b) ((a) < (b) ? (a) : (b)) #define MIN(a, b) ((a) < (b) ? (a) : (b))
@@ -435,16 +425,16 @@ static BMK_benchOutcome_t BMK_benchMemAdvancedNoAlloc(
const char* displayName, const char* displayName,
const BMK_advancedParams_t* adv) const BMK_advancedParams_t* adv)
{ {
size_t const blockSize = size_t const chunkSizeMax =
((adv->blockSize >= 32 && (adv->mode != BMK_decodeOnly)) ((adv->chunkSizeMax >= 32 && (adv->mode != BMK_decodeOnly))
? adv->blockSize ? adv->chunkSizeMax
: srcSize) : srcSize)
+ (!srcSize); /* avoid div by 0 */ + (!srcSize); /* avoid div by 0 */
BMK_benchResult_t benchResult; BMK_benchResult_t benchResult;
size_t const loadedCompressedSize = srcSize; size_t const loadedCompressedSize = srcSize;
size_t cSize = 0; size_t cSize = 0;
double ratio = 0.; double ratio = 0.;
U32 nbBlocks; U32 nbChunks = 0;
assert(cctx != NULL); assert(cctx != NULL);
assert(dctx != NULL); assert(dctx != NULL);
@@ -500,41 +490,42 @@ static BMK_benchOutcome_t BMK_benchMemAdvancedNoAlloc(
} }
} }
/* Init data blocks */ /* Init data chunks */
{ {
const char* srcPtr = (const char*)srcBuffer; const char* srcPtr = (const char*)srcBuffer;
char* cPtr = (char*)compressedBuffer; char* cPtr = (char*)compressedBuffer;
char* resPtr = (char*)(*resultBufferPtr); char* resPtr = (char*)(*resultBufferPtr);
U32 fileNb; U32 fileNb, chunkID;
for (nbBlocks = 0, fileNb = 0; fileNb < nbFiles; fileNb++) { for (chunkID = 0, fileNb = 0; fileNb < nbFiles; fileNb++) {
size_t remaining = fileSizes[fileNb]; size_t remaining = fileSizes[fileNb];
U32 const nbBlocksforThisFile = (adv->mode == BMK_decodeOnly) U32 const nbChunksforThisFile = (adv->mode == BMK_decodeOnly)
? 1 ? 1
: (U32)((remaining + (blockSize - 1)) / blockSize); : (U32)((remaining + (chunkSizeMax - 1)) / chunkSizeMax);
U32 const blockEnd = nbBlocks + nbBlocksforThisFile; U32 const chunkIdEnd = chunkID + nbChunksforThisFile;
for (; nbBlocks < blockEnd; nbBlocks++) { for (; chunkID < chunkIdEnd; chunkID++) {
size_t const thisBlockSize = MIN(remaining, blockSize); size_t const chunkSize = MIN(remaining, chunkSizeMax);
srcPtrs[nbBlocks] = srcPtr; srcPtrs[chunkID] = srcPtr;
srcSizes[nbBlocks] = thisBlockSize; srcSizes[chunkID] = chunkSize;
cPtrs[nbBlocks] = cPtr; cPtrs[chunkID] = cPtr;
cCapacities[nbBlocks] = (adv->mode == BMK_decodeOnly) cCapacities[chunkID] = (adv->mode == BMK_decodeOnly)
? thisBlockSize ? chunkSize
: ZSTD_compressBound(thisBlockSize); : ZSTD_compressBound(chunkSize);
resPtrs[nbBlocks] = resPtr; resPtrs[chunkID] = resPtr;
resSizes[nbBlocks] = (adv->mode == BMK_decodeOnly) resSizes[chunkID] = (adv->mode == BMK_decodeOnly)
? (size_t)ZSTD_findDecompressedSize( ? (size_t)ZSTD_findDecompressedSize(
srcPtr, thisBlockSize) srcPtr, chunkSize)
: thisBlockSize; : chunkSize;
srcPtr += thisBlockSize; srcPtr += chunkSize;
cPtr += cCapacities[nbBlocks]; cPtr += cCapacities[chunkID];
resPtr += thisBlockSize; resPtr += chunkSize;
remaining -= thisBlockSize; remaining -= chunkSize;
if (adv->mode == BMK_decodeOnly) { if (adv->mode == BMK_decodeOnly) {
cSizes[nbBlocks] = thisBlockSize; cSizes[chunkID] = chunkSize;
benchResult.cSize = thisBlockSize; benchResult.cSize = chunkSize;
} }
} }
} }
nbChunks = chunkID;
} }
/* warming up `compressedBuffer` */ /* warming up `compressedBuffer` */
@@ -569,7 +560,7 @@ static BMK_benchOutcome_t BMK_benchMemAdvancedNoAlloc(
cbp.initFn = local_initCCtx; /* BMK_initCCtx */ cbp.initFn = local_initCCtx; /* BMK_initCCtx */
cbp.initPayload = &cctxprep; cbp.initPayload = &cctxprep;
cbp.errorFn = ZSTD_isError; cbp.errorFn = ZSTD_isError;
cbp.blockCount = nbBlocks; cbp.blockCount = nbChunks;
cbp.srcBuffers = srcPtrs; cbp.srcBuffers = srcPtrs;
cbp.srcSizes = srcSizes; cbp.srcSizes = srcSizes;
cbp.dstBuffers = cPtrs; cbp.dstBuffers = cPtrs;
@@ -588,7 +579,7 @@ static BMK_benchOutcome_t BMK_benchMemAdvancedNoAlloc(
dbp.initFn = local_initDCtx; dbp.initFn = local_initDCtx;
dbp.initPayload = &dctxprep; dbp.initPayload = &dctxprep;
dbp.errorFn = ZSTD_isError; dbp.errorFn = ZSTD_isError;
dbp.blockCount = nbBlocks; dbp.blockCount = nbChunks;
dbp.srcBuffers = (const void* const*)cPtrs; dbp.srcBuffers = (const void* const*)cPtrs;
dbp.srcSizes = cSizes; dbp.srcSizes = cSizes;
dbp.dstBuffers = resPtrs; dbp.dstBuffers = resPtrs;
@@ -690,8 +681,7 @@ static BMK_benchOutcome_t BMK_benchMemAdvancedNoAlloc(
} /* while (!(compressionCompleted && decompressionCompleted)) */ } /* while (!(compressionCompleted && decompressionCompleted)) */
/* CRC Checking */ /* CRC Checking */
{ { const BYTE* resultBuffer = (const BYTE*)(*resultBufferPtr);
const BYTE* resultBuffer = (const BYTE*)(*resultBufferPtr);
U64 const crcCheck = XXH64(resultBuffer, srcSize, 0); U64 const crcCheck = XXH64(resultBuffer, srcSize, 0);
if ((adv->mode == BMK_both) && (crcOrig != crcCheck)) { if ((adv->mode == BMK_both) && (crcOrig != crcCheck)) {
size_t u; size_t u;
@@ -704,14 +694,14 @@ static BMK_benchOutcome_t BMK_benchMemAdvancedNoAlloc(
unsigned segNb, bNb, pos; unsigned segNb, bNb, pos;
size_t bacc = 0; size_t bacc = 0;
DISPLAY("Decoding error at pos %u ", (unsigned)u); DISPLAY("Decoding error at pos %u ", (unsigned)u);
for (segNb = 0; segNb < nbBlocks; segNb++) { for (segNb = 0; segNb < nbChunks; segNb++) {
if (bacc + srcSizes[segNb] > u) if (bacc + srcSizes[segNb] > u)
break; break;
bacc += srcSizes[segNb]; bacc += srcSizes[segNb];
} }
pos = (U32)(u - bacc); pos = (U32)(u - bacc);
bNb = pos / (128 KB); bNb = pos / (128 KB);
DISPLAY("(sample %u, block %u, pos %u) \n", DISPLAY("(sample %u, chunk %u, pos %u) \n",
segNb, segNb,
bNb, bNb,
pos); pos);
@@ -795,25 +785,24 @@ BMK_benchOutcome_t BMK_benchMemAdvanced(
int const dstParamsError = int const dstParamsError =
!dstBuffer ^ !dstCapacity; /* must be both NULL or none */ !dstBuffer ^ !dstCapacity; /* must be both NULL or none */
size_t const blockSize = size_t const chunkSize =
((adv->blockSize >= 32 && (adv->mode != BMK_decodeOnly)) ((adv->chunkSizeMax >= 32 && (adv->mode != BMK_decodeOnly))
? adv->blockSize ? adv->chunkSizeMax
: srcSize) : srcSize)
+ (!srcSize) /* avoid div by 0 */; + (!srcSize) /* avoid div by 0 */;
U32 const maxNbBlocks = U32 const nbChunksMax =
(U32)((srcSize + (blockSize - 1)) / blockSize) + nbFiles; (U32)((srcSize + (chunkSize - 1)) / chunkSize) + nbFiles;
/* these are the blockTable parameters, just split up */
const void** const srcPtrs = const void** const srcPtrs =
(const void**)malloc(maxNbBlocks * sizeof(void*)); (const void**)malloc(nbChunksMax * sizeof(void*));
size_t* const srcSizes = (size_t*)malloc(maxNbBlocks * sizeof(size_t)); size_t* const srcSizes = (size_t*)malloc(nbChunksMax * sizeof(size_t));
void** const cPtrs = (void**)malloc(maxNbBlocks * sizeof(void*)); void** const cPtrs = (void**)malloc(nbChunksMax * sizeof(void*));
size_t* const cSizes = (size_t*)malloc(maxNbBlocks * sizeof(size_t)); size_t* const cSizes = (size_t*)malloc(nbChunksMax * sizeof(size_t));
size_t* const cCapacities = (size_t*)malloc(maxNbBlocks * sizeof(size_t)); size_t* const cCapacities = (size_t*)malloc(nbChunksMax * sizeof(size_t));
void** const resPtrs = (void**)malloc(maxNbBlocks * sizeof(void*)); void** const resPtrs = (void**)malloc(nbChunksMax * sizeof(void*));
size_t* const resSizes = (size_t*)malloc(maxNbBlocks * sizeof(size_t)); size_t* const resSizes = (size_t*)malloc(nbChunksMax * sizeof(size_t));
BMK_timedFnState_t* timeStateCompress = BMK_createTimedFnState( BMK_timedFnState_t* timeStateCompress = BMK_createTimedFnState(
adv->nbSeconds * 1000, BMK_RUNTEST_DEFAULT_MS); adv->nbSeconds * 1000, BMK_RUNTEST_DEFAULT_MS);
@@ -825,7 +814,7 @@ BMK_benchOutcome_t BMK_benchMemAdvanced(
const size_t maxCompressedSize = dstCapacity const size_t maxCompressedSize = dstCapacity
? dstCapacity ? dstCapacity
: ZSTD_compressBound(srcSize) + (maxNbBlocks * 1024); : ZSTD_compressBound(srcSize) + (nbChunksMax * 1024);
void* const internalDstBuffer = void* const internalDstBuffer =
dstBuffer ? NULL : malloc(maxCompressedSize); dstBuffer ? NULL : malloc(maxCompressedSize);
@@ -964,12 +953,12 @@ static int BMK_benchCLevels(
} }
if (displayLevel == 1 && !adv->additionalParam) /* --quiet mode */ if (displayLevel == 1 && !adv->additionalParam) /* --quiet mode */
OUTPUT("bench %s %s: input %u bytes, %u seconds, %u KB blocks\n", OUTPUT("bench %s %s: input %u bytes, %u seconds, %u KB chunks\n",
ZSTD_VERSION_STRING, ZSTD_VERSION_STRING,
ZSTD_GIT_COMMIT_STRING, ZSTD_GIT_COMMIT_STRING,
(unsigned)benchedSize, (unsigned)benchedSize,
adv->nbSeconds, adv->nbSeconds,
(unsigned)(adv->blockSize >> 10)); (unsigned)(adv->chunkSizeMax >> 10));
for (level = startCLevel; level <= endCLevel; level++) { for (level = startCLevel; level <= endCLevel; level++) {
BMK_benchOutcome_t res = BMK_benchMemAdvanced( BMK_benchOutcome_t res = BMK_benchMemAdvanced(
@@ -1000,7 +989,7 @@ int BMK_syntheticTest(
{ {
char nameBuff[20] = { 0 }; char nameBuff[20] = { 0 };
const char* name = nameBuff; const char* name = nameBuff;
size_t const benchedSize = adv->blockSize ? adv->blockSize : 10000000; size_t const benchedSize = adv->chunkSizeMax ? adv->chunkSizeMax : 10000000;
/* Memory allocation */ /* Memory allocation */
void* const srcBuffer = malloc(benchedSize); void* const srcBuffer = malloc(benchedSize);
programs/benchzstd.h
+1 -1
View File
@@ -94,7 +94,7 @@ typedef enum {
typedef struct { typedef struct {
BMK_mode_t mode; /* 0: both, 1: compress only 2: decode only */ BMK_mode_t mode; /* 0: both, 1: compress only 2: decode only */
unsigned nbSeconds; /* default timing is in nbSeconds */ unsigned nbSeconds; /* default timing is in nbSeconds */
size_t blockSize; /* Maximum size of each independent chunk */ size_t chunkSizeMax; /* Maximum size of each independent chunk */
size_t targetCBlockSize;/* Approximative size of compressed blocks */ size_t targetCBlockSize;/* Approximative size of compressed blocks */
int nbWorkers; /* multithreading */ int nbWorkers; /* multithreading */
unsigned realTime; /* real time priority */ unsigned realTime; /* real time priority */
programs/zstdcli.c
+1 -1
View File
@@ -1414,7 +1414,7 @@ int main(int argCount, const char* argv[])
DISPLAYLEVEL(1, "benchmark mode is only compatible with zstd format \n"); DISPLAYLEVEL(1, "benchmark mode is only compatible with zstd format \n");
CLEAN_RETURN(1); CLEAN_RETURN(1);
} }
benchParams.blockSize = chunkSize; benchParams.chunkSizeMax = chunkSize;
benchParams.targetCBlockSize = targetCBlockSize; benchParams.targetCBlockSize = targetCBlockSize;
benchParams.nbWorkers = (int)nbWorkers; benchParams.nbWorkers = (int)nbWorkers;
benchParams.realTime = (unsigned)setRealTimePrio; benchParams.realTime = (unsigned)setRealTimePrio;