From 8accfa7fcc3da7d1f81a4b3e879293fd393a1099 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Fri, 17 Nov 2017 00:02:37 -0800 Subject: [PATCH 1/4] bench: realTime is a global parameter like most parameters not directly related to compression --- programs/bench.c | 35 ++++++++++++++++++++++------------- programs/bench.h | 3 ++- programs/zstdcli.c | 3 ++- 3 files changed, 26 insertions(+), 15 deletions(-) diff --git a/programs/bench.c b/programs/bench.c index 6525f3371..c61273936 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -129,6 +129,12 @@ void BMK_setNbThreads(unsigned nbThreads) { #endif g_nbThreads = nbThreads; } + +static U32 g_realTime = 0; +void BMK_setRealTime(unsigned priority) { + g_realTime = (priority>0); +} + static U32 g_ldmFlag = 0; void BMK_setLdmFlag(unsigned ldmFlag) { g_ldmFlag = ldmFlag; @@ -480,7 +486,7 @@ static void BMK_benchCLevel(void* srcBuffer, size_t benchedSize, const char* displayName, int cLevel, int cLevelLast, const size_t* fileSizes, unsigned nbFiles, const void* dictBuffer, size_t dictBufferSize, - ZSTD_compressionParameters *compressionParams, int setRealTimePrio) + ZSTD_compressionParameters *compressionParams) { int l; @@ -488,8 +494,8 @@ static void BMK_benchCLevel(void* srcBuffer, size_t benchedSize, if (!pch) pch = strrchr(displayName, '/'); /* Linux */ if (pch) displayName = pch+1; - if (setRealTimePrio) { - DISPLAYLEVEL(2, "Note : switching to a real-time priority \n"); + if (g_realTime) { + DISPLAYLEVEL(2, "Note : switching to real-time priority \n"); SET_REALTIME_PRIORITY; } @@ -545,7 +551,7 @@ static void BMK_loadFiles(void* buffer, size_t bufferSize, } static void BMK_benchFileTable(const char** fileNamesTable, unsigned nbFiles, const char* dictFileName, int cLevel, - int cLevelLast, ZSTD_compressionParameters *compressionParams, int setRealTimePrio) + int cLevelLast, ZSTD_compressionParameters *compressionParams) { void* srcBuffer; size_t benchedSize; @@ -583,11 +589,11 @@ static void BMK_benchFileTable(const char** fileNamesTable, unsigned nbFiles, co /* Bench */ snprintf (mfName, sizeof(mfName), " %u files", nbFiles); - { const char* displayName = (nbFiles > 1) ? mfName : fileNamesTable[0]; + { const char* const displayName = (nbFiles > 1) ? mfName : fileNamesTable[0]; BMK_benchCLevel(srcBuffer, benchedSize, displayName, cLevel, cLevelLast, fileSizes, nbFiles, - dictBuffer, dictBufferSize, compressionParams, setRealTimePrio); + dictBuffer, dictBufferSize, compressionParams); } /* clean up */ @@ -597,7 +603,7 @@ static void BMK_benchFileTable(const char** fileNamesTable, unsigned nbFiles, co } -static void BMK_syntheticTest(int cLevel, int cLevelLast, double compressibility, ZSTD_compressionParameters* compressionParams, int setRealTimePrio) +static void BMK_syntheticTest(int cLevel, int cLevelLast, double compressibility, ZSTD_compressionParameters* compressionParams) { char name[20] = {0}; size_t benchedSize = 10000000; @@ -611,15 +617,17 @@ static void BMK_syntheticTest(int cLevel, int cLevelLast, double compressibility /* Bench */ snprintf (name, sizeof(name), "Synthetic %2u%%", (unsigned)(compressibility*100)); - BMK_benchCLevel(srcBuffer, benchedSize, name, cLevel, cLevelLast, &benchedSize, 1, NULL, 0, compressionParams, setRealTimePrio); + BMK_benchCLevel(srcBuffer, benchedSize, name, cLevel, cLevelLast, &benchedSize, 1, NULL, 0, compressionParams); /* clean up */ free(srcBuffer); } -int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles, const char* dictFileName, - int cLevel, int cLevelLast, ZSTD_compressionParameters* compressionParams, int setRealTimePrio) +int BMK_benchFiles(const char** const fileNamesTable, unsigned const nbFiles, + const char* dictFileName, + int cLevel, int cLevelLast, + ZSTD_compressionParameters* compressionParams) { double const compressibility = (double)g_compressibilityDefault / 100; @@ -627,11 +635,12 @@ int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles, const char* di if (cLevel > ZSTD_maxCLevel()) cLevel = ZSTD_maxCLevel(); if (cLevelLast > ZSTD_maxCLevel()) cLevelLast = ZSTD_maxCLevel(); if (cLevelLast < cLevel) cLevelLast = cLevel; - if (cLevelLast > cLevel) DISPLAYLEVEL(2, "Benchmarking levels from %d to %d\n", cLevel, cLevelLast); + if (cLevelLast > cLevel) + DISPLAYLEVEL(2, "Benchmarking levels from %d to %d\n", cLevel, cLevelLast); if (nbFiles == 0) - BMK_syntheticTest(cLevel, cLevelLast, compressibility, compressionParams, setRealTimePrio); + BMK_syntheticTest(cLevel, cLevelLast, compressibility, compressionParams); else - BMK_benchFileTable(fileNamesTable, nbFiles, dictFileName, cLevel, cLevelLast, compressionParams, setRealTimePrio); + BMK_benchFileTable(fileNamesTable, nbFiles, dictFileName, cLevel, cLevelLast, compressionParams); return 0; } diff --git a/programs/bench.h b/programs/bench.h index 82bb34569..f4d415488 100644 --- a/programs/bench.h +++ b/programs/bench.h @@ -17,12 +17,13 @@ #include "zstd.h" /* ZSTD_compressionParameters */ int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles,const char* dictFileName, - int cLevel, int cLevelLast, ZSTD_compressionParameters* compressionParams, int setRealTimePrio); + int cLevel, int cLevelLast, ZSTD_compressionParameters* compressionParams); /* Set Parameters */ void BMK_setNbSeconds(unsigned nbLoops); void BMK_setBlockSize(size_t blockSize); void BMK_setNbThreads(unsigned nbThreads); +void BMK_setRealTime(unsigned priority); void BMK_setNotificationLevel(unsigned level); void BMK_setAdditionalParam(int additionalParam); void BMK_setDecodeOnlyMode(unsigned decodeFlag); diff --git a/programs/zstdcli.c b/programs/zstdcli.c index 3f8367341..d5cb6285c 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -753,6 +753,7 @@ int main(int argCount, const char* argv[]) BMK_setNotificationLevel(g_displayLevel); BMK_setBlockSize(blockSize); BMK_setNbThreads(nbThreads); + BMK_setRealTime(setRealTimePrio); BMK_setNbSeconds(bench_nbSeconds); BMK_setLdmFlag(ldmFlag); BMK_setLdmMinMatch(g_ldmMinMatch); @@ -763,7 +764,7 @@ int main(int argCount, const char* argv[]) if (g_ldmHashEveryLog != LDM_PARAM_DEFAULT) { BMK_setLdmHashEveryLog(g_ldmHashEveryLog); } - BMK_benchFiles(filenameTable, filenameIdx, dictFileName, cLevel, cLevelLast, &compressionParams, setRealTimePrio); + BMK_benchFiles(filenameTable, filenameIdx, dictFileName, cLevel, cLevelLast, &compressionParams); #endif (void)bench_nbSeconds; (void)blockSize; (void)setRealTimePrio; goto _end; From d898fb7ba60611a3c293c8b8098960263e0bd7db Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Fri, 17 Nov 2017 00:22:55 -0800 Subject: [PATCH 2/4] bench: added cli command `-S` to benchmark multiple files separately Currently, all files are joined by default, they are compressed separately but benchmarked together, providing a single final result. Benchmarking files separately make it possible to accurately measure difference for each file. This is expected to be useful while tuning optimal parser. --- programs/bench.c | 47 ++++++++++++++++++++++++++++++++-------------- programs/bench.h | 1 + programs/util.h | 2 +- programs/zstdcli.c | 8 ++++++++ 4 files changed, 43 insertions(+), 15 deletions(-) diff --git a/programs/bench.c b/programs/bench.c index c61273936..e28bed747 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -135,6 +135,11 @@ void BMK_setRealTime(unsigned priority) { g_realTime = (priority>0); } +static U32 g_separateFiles = 0; +void BMK_setSeparateFiles(unsigned separate) { + g_separateFiles = (separate>0); +} + static U32 g_ldmFlag = 0; void BMK_setLdmFlag(unsigned ldmFlag) { g_ldmFlag = ldmFlag; @@ -482,11 +487,11 @@ static size_t BMK_findMaxMem(U64 requiredMem) return (size_t)(requiredMem); } -static void BMK_benchCLevel(void* srcBuffer, size_t benchedSize, +static void BMK_benchCLevel(const void* srcBuffer, size_t benchedSize, const char* displayName, int cLevel, int cLevelLast, const size_t* fileSizes, unsigned nbFiles, const void* dictBuffer, size_t dictBufferSize, - ZSTD_compressionParameters *compressionParams) + const ZSTD_compressionParameters* const compressionParams) { int l; @@ -518,7 +523,7 @@ static void BMK_benchCLevel(void* srcBuffer, size_t benchedSize, At most, fills `buffer` entirely */ static void BMK_loadFiles(void* buffer, size_t bufferSize, size_t* fileSizes, - const char** fileNamesTable, unsigned nbFiles) + const char* const * const fileNamesTable, unsigned nbFiles) { size_t pos = 0, totalSize = 0; unsigned n; @@ -550,16 +555,17 @@ static void BMK_loadFiles(void* buffer, size_t bufferSize, if (totalSize == 0) EXM_THROW(12, "no data to bench"); } -static void BMK_benchFileTable(const char** fileNamesTable, unsigned nbFiles, const char* dictFileName, int cLevel, - int cLevelLast, ZSTD_compressionParameters *compressionParams) +static void BMK_benchFileTable(const char* const * const fileNamesTable, unsigned const nbFiles, + const char* const dictFileName, + int const cLevel, int const cLevelLast, + const ZSTD_compressionParameters* const compressionParams) { void* srcBuffer; size_t benchedSize; void* dictBuffer = NULL; size_t dictBufferSize = 0; - size_t* fileSizes = (size_t*)malloc(nbFiles * sizeof(size_t)); + size_t* const fileSizes = (size_t*)malloc(nbFiles * sizeof(size_t)); U64 const totalSizeToLoad = UTIL_getTotalFileSize(fileNamesTable, nbFiles); - char mfName[20] = {0}; if (!fileSizes) EXM_THROW(12, "not enough memory for fileSizes"); @@ -588,13 +594,26 @@ static void BMK_benchFileTable(const char** fileNamesTable, unsigned nbFiles, co BMK_loadFiles(srcBuffer, benchedSize, fileSizes, fileNamesTable, nbFiles); /* Bench */ - snprintf (mfName, sizeof(mfName), " %u files", nbFiles); - { const char* const displayName = (nbFiles > 1) ? mfName : fileNamesTable[0]; - BMK_benchCLevel(srcBuffer, benchedSize, - displayName, cLevel, cLevelLast, - fileSizes, nbFiles, - dictBuffer, dictBufferSize, compressionParams); - } + if (g_separateFiles) { + const BYTE* srcPtr = (const BYTE*)srcBuffer; + U32 fileNb; + for (fileNb=0; fileNb 1) ? mfName : fileNamesTable[0]; + BMK_benchCLevel(srcBuffer, benchedSize, + displayName, cLevel, cLevelLast, + fileSizes, nbFiles, + dictBuffer, dictBufferSize, compressionParams); + } } /* clean up */ free(srcBuffer); diff --git a/programs/bench.h b/programs/bench.h index f4d415488..597f25036 100644 --- a/programs/bench.h +++ b/programs/bench.h @@ -25,6 +25,7 @@ void BMK_setBlockSize(size_t blockSize); void BMK_setNbThreads(unsigned nbThreads); void BMK_setRealTime(unsigned priority); void BMK_setNotificationLevel(unsigned level); +void BMK_setSeparateFiles(unsigned separate); void BMK_setAdditionalParam(int additionalParam); void BMK_setDecodeOnlyMode(unsigned decodeFlag); void BMK_setLdmFlag(unsigned ldmFlag); diff --git a/programs/util.h b/programs/util.h index c5e4365d0..e44d7459d 100644 --- a/programs/util.h +++ b/programs/util.h @@ -336,7 +336,7 @@ UTIL_STATIC U64 UTIL_getFileSize(const char* infilename) } -UTIL_STATIC U64 UTIL_getTotalFileSize(const char** fileNamesTable, unsigned nbFiles) +UTIL_STATIC U64 UTIL_getTotalFileSize(const char* const * const fileNamesTable, unsigned nbFiles) { U64 total = 0; int error = 0; diff --git a/programs/zstdcli.c b/programs/zstdcli.c index d5cb6285c..91af49e24 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -377,6 +377,7 @@ int main(int argCount, const char* argv[]) lastCommand = 0, nbThreads = 1, setRealTimePrio = 0, + separateFiles = 0, ldmFlag = 0; unsigned bench_nbSeconds = 3; /* would be better if this value was synchronized from bench */ size_t blockSize = 0; @@ -633,6 +634,12 @@ int main(int argCount, const char* argv[]) blockSize = readU32FromChar(&argument); break; + /* benchmark files separately (hidden option) */ + case 'S': + argument++; + separateFiles = 1; + break; + #endif /* ZSTD_NOBENCH */ /* nb of threads (hidden option) */ @@ -751,6 +758,7 @@ int main(int argCount, const char* argv[]) if (operation==zom_bench) { #ifndef ZSTD_NOBENCH BMK_setNotificationLevel(g_displayLevel); + BMK_setSeparateFiles(separateFiles); BMK_setBlockSize(blockSize); BMK_setNbThreads(nbThreads); BMK_setRealTime(setRealTimePrio); From 5b957ba8997e0ae5b32f8125ea02a8e2205aa3a9 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Fri, 17 Nov 2017 01:21:40 -0800 Subject: [PATCH 3/4] minor interface adjustments --- Makefile | 5 ----- programs/bench.c | 6 +++--- programs/bench.h | 4 ++-- 3 files changed, 5 insertions(+), 10 deletions(-) diff --git a/Makefile b/Makefile index 9f4b1fe4d..9c2657052 100644 --- a/Makefile +++ b/Makefile @@ -130,11 +130,6 @@ uninstall: travis-install: $(MAKE) install PREFIX=~/install_test_dir -.PHONY: gppbuild -gppbuild: clean - g++ -v - CC=g++ $(MAKE) -C programs all CFLAGS="-O3 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Werror" - .PHONY: gcc5build gcc5build: clean gcc-5 -v diff --git a/programs/bench.c b/programs/bench.c index e28bed747..27952169f 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -622,7 +622,7 @@ static void BMK_benchFileTable(const char* const * const fileNamesTable, unsigne } -static void BMK_syntheticTest(int cLevel, int cLevelLast, double compressibility, ZSTD_compressionParameters* compressionParams) +static void BMK_syntheticTest(int cLevel, int cLevelLast, double compressibility, const ZSTD_compressionParameters* compressionParams) { char name[20] = {0}; size_t benchedSize = 10000000; @@ -643,10 +643,10 @@ static void BMK_syntheticTest(int cLevel, int cLevelLast, double compressibility } -int BMK_benchFiles(const char** const fileNamesTable, unsigned const nbFiles, +int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles, const char* dictFileName, int cLevel, int cLevelLast, - ZSTD_compressionParameters* compressionParams) + const ZSTD_compressionParameters* compressionParams) { double const compressibility = (double)g_compressibilityDefault / 100; diff --git a/programs/bench.h b/programs/bench.h index 597f25036..d2ec94ee3 100644 --- a/programs/bench.h +++ b/programs/bench.h @@ -16,8 +16,8 @@ #define ZSTD_STATIC_LINKING_ONLY /* ZSTD_compressionParameters */ #include "zstd.h" /* ZSTD_compressionParameters */ -int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles,const char* dictFileName, - int cLevel, int cLevelLast, ZSTD_compressionParameters* compressionParams); +int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles, const char* dictFileName, + int cLevel, int cLevelLast, const ZSTD_compressionParameters* compressionParams); /* Set Parameters */ void BMK_setNbSeconds(unsigned nbLoops); From 49a445e65d6eb71e4cca371ea37560852362db04 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Fri, 17 Nov 2017 02:14:16 -0800 Subject: [PATCH 4/4] fixed circle-ci script --- circle.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/circle.yml b/circle.yml index 5bc0ce643..b488c1a7b 100644 --- a/circle.yml +++ b/circle.yml @@ -24,7 +24,7 @@ test: : parallel: true - ? | - if [[ "$CIRCLE_NODE_INDEX" == "0" ]] ; then make gppbuild && make clean; fi && + if [[ "$CIRCLE_NODE_INDEX" == "0" ]] ; then make cxxtest && make clean; fi && if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make gcc5build && make clean; fi : parallel: true