ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #3480 from facebook/benchFail

Browse Source 
return error code when benchmark fails
This commit is contained in:
Yann Collet
2023-02-07 12:42:47 -08:00
committed by GitHub
parent 54a173e9fc 6740f8f0b8
commit df21aceaa7
3 changed files with 25 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files
programs/benchzstd.c
+15 -10
View File
@@ -327,26 +327,31 @@ BMK_benchMemAdvancedNoAlloc(
/* init */ /* init */
memset(&benchResult, 0, sizeof(benchResult)); memset(&benchResult, 0, sizeof(benchResult));
if (strlen(displayName)>17) displayName += strlen(displayName) - 17; /* display last 17 characters */ if (strlen(displayName)>17) displayName += strlen(displayName) - 17; /* display last 17 characters */
if (adv->mode == BMK_decodeOnly) { /* benchmark only decompression : source must be already compressed */ if (adv->mode == BMK_decodeOnly) {
/* benchmark only decompression : source must be already compressed */
const char* srcPtr = (const char*)srcBuffer; const char* srcPtr = (const char*)srcBuffer;
U64 totalDSize64 = 0; U64 totalDSize64 = 0;
U32 fileNb; U32 fileNb;
for (fileNb=0; fileNb<nbFiles; fileNb++) { for (fileNb=0; fileNb<nbFiles; fileNb++) {
U64 const fSize64 = ZSTD_findDecompressedSize(srcPtr, fileSizes[fileNb]); U64 const fSize64 = ZSTD_findDecompressedSize(srcPtr, fileSizes[fileNb]);
if (fSize64==0) RETURN_ERROR(32, BMK_benchOutcome_t, "Impossible to determine original size "); if (fSize64 == ZSTD_CONTENTSIZE_UNKNOWN) {
RETURN_ERROR(32, BMK_benchOutcome_t, "Decompressed size cannot be determined: cannot benchmark");
}
if (fSize64 == ZSTD_CONTENTSIZE_ERROR) {
RETURN_ERROR(32, BMK_benchOutcome_t, "Error while trying to assess decompressed size: data may be invalid");
}
totalDSize64 += fSize64; totalDSize64 += fSize64;
srcPtr += fileSizes[fileNb]; srcPtr += fileSizes[fileNb];
} }
{ size_t const decodedSize = (size_t)totalDSize64; { size_t const decodedSize = (size_t)totalDSize64;
assert((U64)decodedSize == totalDSize64); /* check overflow */ assert((U64)decodedSize == totalDSize64); /* check overflow */
free(*resultBufferPtr); free(*resultBufferPtr);
if (totalDSize64 > decodedSize) { /* size_t overflow */
RETURN_ERROR(32, BMK_benchOutcome_t, "decompressed size is too large for local system");
}
*resultBufferPtr = malloc(decodedSize); *resultBufferPtr = malloc(decodedSize);
if (!(*resultBufferPtr)) { if (!(*resultBufferPtr)) {
RETURN_ERROR(33, BMK_benchOutcome_t, "not enough memory"); RETURN_ERROR(33, BMK_benchOutcome_t, "allocation error: not enough memory");
}
if (totalDSize64 > decodedSize) { /* size_t overflow */
free(*resultBufferPtr);
RETURN_ERROR(32, BMK_benchOutcome_t, "original size is too large");
} }
cSize = srcSize; cSize = srcSize;
srcSize = decodedSize; srcSize = decodedSize;
@@ -446,7 +451,7 @@ BMK_benchMemAdvancedNoAlloc(
BMK_runOutcome_t const cOutcome = BMK_benchTimedFn( timeStateCompress, cbp); BMK_runOutcome_t const cOutcome = BMK_benchTimedFn( timeStateCompress, cbp);
if (!BMK_isSuccessful_runOutcome(cOutcome)) { if (!BMK_isSuccessful_runOutcome(cOutcome)) {
return BMK_benchOutcome_error(); RETURN_ERROR(30, BMK_benchOutcome_t, "compression error");
} }
{ BMK_runTime_t const cResult = BMK_extract_runTime(cOutcome); { BMK_runTime_t const cResult = BMK_extract_runTime(cOutcome);
@@ -474,7 +479,7 @@ BMK_benchMemAdvancedNoAlloc(
BMK_runOutcome_t const dOutcome = BMK_benchTimedFn(timeStateDecompress, dbp); BMK_runOutcome_t const dOutcome = BMK_benchTimedFn(timeStateDecompress, dbp);
if(!BMK_isSuccessful_runOutcome(dOutcome)) { if(!BMK_isSuccessful_runOutcome(dOutcome)) {
return BMK_benchOutcome_error(); RETURN_ERROR(30, BMK_benchOutcome_t, "decompression error");
} }
{ BMK_runTime_t const dResult = BMK_extract_runTime(dOutcome); { BMK_runTime_t const dResult = BMK_extract_runTime(dOutcome);
@@ -598,7 +603,7 @@ BMK_benchOutcome_t BMK_benchMemAdvanced(const void* srcBuffer, size_t srcSize,
void* resultBuffer = srcSize ? malloc(srcSize) : NULL; void* resultBuffer = srcSize ? malloc(srcSize) : NULL;
int allocationincomplete = !srcPtrs || !srcSizes || !cPtrs || int const allocationincomplete = !srcPtrs || !srcSizes || !cPtrs ||
!cSizes || !cCapacities || !resPtrs || !resSizes || !cSizes || !cCapacities || !resPtrs || !resSizes ||
!timeStateCompress || !timeStateDecompress || !timeStateCompress || !timeStateDecompress ||
!cctx || !dctx || !cctx || !dctx ||
programs/zstdcli.c
+8 -5
View File
@@ -1370,7 +1370,7 @@ int main(int argCount, const char* argv[])
benchParams.ldmFlag = ldmFlag; benchParams.ldmFlag = ldmFlag;
benchParams.ldmMinMatch = (int)g_ldmMinMatch; benchParams.ldmMinMatch = (int)g_ldmMinMatch;
benchParams.ldmHashLog = (int)g_ldmHashLog; benchParams.ldmHashLog = (int)g_ldmHashLog;
benchParams.useRowMatchFinder = useRowMatchFinder; benchParams.useRowMatchFinder = (int)useRowMatchFinder;
if (g_ldmBucketSizeLog != LDM_PARAM_DEFAULT) { if (g_ldmBucketSizeLog != LDM_PARAM_DEFAULT) {
benchParams.ldmBucketSizeLog = (int)g_ldmBucketSizeLog; benchParams.ldmBucketSizeLog = (int)g_ldmBucketSizeLog;
} }
@@ -1391,15 +1391,18 @@ int main(int argCount, const char* argv[])
int c; int c;
DISPLAYLEVEL(3, "Benchmarking %s \n", filenames->fileNames[i]); DISPLAYLEVEL(3, "Benchmarking %s \n", filenames->fileNames[i]);
for(c = cLevel; c <= cLevelLast; c++) { for(c = cLevel; c <= cLevelLast; c++) {
BMK_benchFilesAdvanced(&filenames->fileNames[i], 1, dictFileName, c, &compressionParams, g_displayLevel, &benchParams); BMK_benchOutcome_t const bo = BMK_benchFilesAdvanced(&filenames->fileNames[i], 1, dictFileName, c, &compressionParams, g_displayLevel, &benchParams);
if (!BMK_isSuccessful_benchOutcome(bo)) return 1;
} } } }
} else { } else {
for(; cLevel <= cLevelLast; cLevel++) { for(; cLevel <= cLevelLast; cLevel++) {
BMK_benchFilesAdvanced(filenames->fileNames, (unsigned)filenames->tableSize, dictFileName, cLevel, &compressionParams, g_displayLevel, &benchParams); BMK_benchOutcome_t const bo = BMK_benchFilesAdvanced(filenames->fileNames, (unsigned)filenames->tableSize, dictFileName, cLevel, &compressionParams, g_displayLevel, &benchParams);
if (!BMK_isSuccessful_benchOutcome(bo)) return 1;
} } } }
} else { } else {
for(; cLevel <= cLevelLast; cLevel++) { for(; cLevel <= cLevelLast; cLevel++) {
BMK_syntheticTest(cLevel, compressibility, &compressionParams, g_displayLevel, &benchParams); BMK_benchOutcome_t const bo = BMK_syntheticTest(cLevel, compressibility, &compressionParams, g_displayLevel, &benchParams);
if (!BMK_isSuccessful_benchOutcome(bo)) return 1;
} } } }
#else #else
@@ -1545,7 +1548,7 @@ int main(int argCount, const char* argv[])
if (g_ldmBucketSizeLog != LDM_PARAM_DEFAULT) FIO_setLdmBucketSizeLog(prefs, (int)g_ldmBucketSizeLog); if (g_ldmBucketSizeLog != LDM_PARAM_DEFAULT) FIO_setLdmBucketSizeLog(prefs, (int)g_ldmBucketSizeLog);
if (g_ldmHashRateLog != LDM_PARAM_DEFAULT) FIO_setLdmHashRateLog(prefs, (int)g_ldmHashRateLog); if (g_ldmHashRateLog != LDM_PARAM_DEFAULT) FIO_setLdmHashRateLog(prefs, (int)g_ldmHashRateLog);
FIO_setAdaptiveMode(prefs, adapt); FIO_setAdaptiveMode(prefs, adapt);
FIO_setUseRowMatchFinder(prefs, useRowMatchFinder); FIO_setUseRowMatchFinder(prefs, (int)useRowMatchFinder);
FIO_setAdaptMin(prefs, adaptMin); FIO_setAdaptMin(prefs, adaptMin);
FIO_setAdaptMax(prefs, adaptMax); FIO_setAdaptMax(prefs, adaptMax);
FIO_setRsyncable(prefs, rsyncable); FIO_setRsyncable(prefs, rsyncable);
tests/playTests.sh
+2
View File
@@ -1217,6 +1217,8 @@ zstd -rqi0b1e2 tmp1
println "benchmark decompression only" println "benchmark decompression only"
zstd -f tmp1 zstd -f tmp1
zstd -b -d -i0 tmp1.zst zstd -b -d -i0 tmp1.zst
println "benchmark can fail - decompression on invalid data"
zstd -b -d -i0 tmp1 && die "invalid .zst data => benchmark should have failed"
GZIPMODE=1 GZIPMODE=1
zstd --format=gzip -V || GZIPMODE=0 zstd --format=gzip -V || GZIPMODE=0