diff --git a/Makefile b/Makefile index 9f4b1fe4d..9c2657052 100644 --- a/Makefile +++ b/Makefile @@ -130,11 +130,6 @@ uninstall: travis-install: $(MAKE) install PREFIX=~/install_test_dir -.PHONY: gppbuild -gppbuild: clean - g++ -v - CC=g++ $(MAKE) -C programs all CFLAGS="-O3 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Werror" - .PHONY: gcc5build gcc5build: clean gcc-5 -v diff --git a/circle.yml b/circle.yml index 5bc0ce643..b488c1a7b 100644 --- a/circle.yml +++ b/circle.yml @@ -24,7 +24,7 @@ test: : parallel: true - ? | - if [[ "$CIRCLE_NODE_INDEX" == "0" ]] ; then make gppbuild && make clean; fi && + if [[ "$CIRCLE_NODE_INDEX" == "0" ]] ; then make cxxtest && make clean; fi && if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make gcc5build && make clean; fi : parallel: true diff --git a/programs/bench.c b/programs/bench.c index 6525f3371..27952169f 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -129,6 +129,17 @@ void BMK_setNbThreads(unsigned nbThreads) { #endif g_nbThreads = nbThreads; } + +static U32 g_realTime = 0; +void BMK_setRealTime(unsigned priority) { + g_realTime = (priority>0); +} + +static U32 g_separateFiles = 0; +void BMK_setSeparateFiles(unsigned separate) { + g_separateFiles = (separate>0); +} + static U32 g_ldmFlag = 0; void BMK_setLdmFlag(unsigned ldmFlag) { g_ldmFlag = ldmFlag; @@ -476,11 +487,11 @@ static size_t BMK_findMaxMem(U64 requiredMem) return (size_t)(requiredMem); } -static void BMK_benchCLevel(void* srcBuffer, size_t benchedSize, +static void BMK_benchCLevel(const void* srcBuffer, size_t benchedSize, const char* displayName, int cLevel, int cLevelLast, const size_t* fileSizes, unsigned nbFiles, const void* dictBuffer, size_t dictBufferSize, - ZSTD_compressionParameters *compressionParams, int setRealTimePrio) + const ZSTD_compressionParameters* const compressionParams) { int l; @@ -488,8 +499,8 @@ static void BMK_benchCLevel(void* srcBuffer, size_t benchedSize, if (!pch) pch = strrchr(displayName, '/'); /* Linux */ if (pch) displayName = pch+1; - if (setRealTimePrio) { - DISPLAYLEVEL(2, "Note : switching to a real-time priority \n"); + if (g_realTime) { + DISPLAYLEVEL(2, "Note : switching to real-time priority \n"); SET_REALTIME_PRIORITY; } @@ -512,7 +523,7 @@ static void BMK_benchCLevel(void* srcBuffer, size_t benchedSize, At most, fills `buffer` entirely */ static void BMK_loadFiles(void* buffer, size_t bufferSize, size_t* fileSizes, - const char** fileNamesTable, unsigned nbFiles) + const char* const * const fileNamesTable, unsigned nbFiles) { size_t pos = 0, totalSize = 0; unsigned n; @@ -544,16 +555,17 @@ static void BMK_loadFiles(void* buffer, size_t bufferSize, if (totalSize == 0) EXM_THROW(12, "no data to bench"); } -static void BMK_benchFileTable(const char** fileNamesTable, unsigned nbFiles, const char* dictFileName, int cLevel, - int cLevelLast, ZSTD_compressionParameters *compressionParams, int setRealTimePrio) +static void BMK_benchFileTable(const char* const * const fileNamesTable, unsigned const nbFiles, + const char* const dictFileName, + int const cLevel, int const cLevelLast, + const ZSTD_compressionParameters* const compressionParams) { void* srcBuffer; size_t benchedSize; void* dictBuffer = NULL; size_t dictBufferSize = 0; - size_t* fileSizes = (size_t*)malloc(nbFiles * sizeof(size_t)); + size_t* const fileSizes = (size_t*)malloc(nbFiles * sizeof(size_t)); U64 const totalSizeToLoad = UTIL_getTotalFileSize(fileNamesTable, nbFiles); - char mfName[20] = {0}; if (!fileSizes) EXM_THROW(12, "not enough memory for fileSizes"); @@ -582,13 +594,26 @@ static void BMK_benchFileTable(const char** fileNamesTable, unsigned nbFiles, co BMK_loadFiles(srcBuffer, benchedSize, fileSizes, fileNamesTable, nbFiles); /* Bench */ - snprintf (mfName, sizeof(mfName), " %u files", nbFiles); - { const char* displayName = (nbFiles > 1) ? mfName : fileNamesTable[0]; - BMK_benchCLevel(srcBuffer, benchedSize, - displayName, cLevel, cLevelLast, - fileSizes, nbFiles, - dictBuffer, dictBufferSize, compressionParams, setRealTimePrio); - } + if (g_separateFiles) { + const BYTE* srcPtr = (const BYTE*)srcBuffer; + U32 fileNb; + for (fileNb=0; fileNb 1) ? mfName : fileNamesTable[0]; + BMK_benchCLevel(srcBuffer, benchedSize, + displayName, cLevel, cLevelLast, + fileSizes, nbFiles, + dictBuffer, dictBufferSize, compressionParams); + } } /* clean up */ free(srcBuffer); @@ -597,7 +622,7 @@ static void BMK_benchFileTable(const char** fileNamesTable, unsigned nbFiles, co } -static void BMK_syntheticTest(int cLevel, int cLevelLast, double compressibility, ZSTD_compressionParameters* compressionParams, int setRealTimePrio) +static void BMK_syntheticTest(int cLevel, int cLevelLast, double compressibility, const ZSTD_compressionParameters* compressionParams) { char name[20] = {0}; size_t benchedSize = 10000000; @@ -611,15 +636,17 @@ static void BMK_syntheticTest(int cLevel, int cLevelLast, double compressibility /* Bench */ snprintf (name, sizeof(name), "Synthetic %2u%%", (unsigned)(compressibility*100)); - BMK_benchCLevel(srcBuffer, benchedSize, name, cLevel, cLevelLast, &benchedSize, 1, NULL, 0, compressionParams, setRealTimePrio); + BMK_benchCLevel(srcBuffer, benchedSize, name, cLevel, cLevelLast, &benchedSize, 1, NULL, 0, compressionParams); /* clean up */ free(srcBuffer); } -int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles, const char* dictFileName, - int cLevel, int cLevelLast, ZSTD_compressionParameters* compressionParams, int setRealTimePrio) +int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles, + const char* dictFileName, + int cLevel, int cLevelLast, + const ZSTD_compressionParameters* compressionParams) { double const compressibility = (double)g_compressibilityDefault / 100; @@ -627,11 +654,12 @@ int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles, const char* di if (cLevel > ZSTD_maxCLevel()) cLevel = ZSTD_maxCLevel(); if (cLevelLast > ZSTD_maxCLevel()) cLevelLast = ZSTD_maxCLevel(); if (cLevelLast < cLevel) cLevelLast = cLevel; - if (cLevelLast > cLevel) DISPLAYLEVEL(2, "Benchmarking levels from %d to %d\n", cLevel, cLevelLast); + if (cLevelLast > cLevel) + DISPLAYLEVEL(2, "Benchmarking levels from %d to %d\n", cLevel, cLevelLast); if (nbFiles == 0) - BMK_syntheticTest(cLevel, cLevelLast, compressibility, compressionParams, setRealTimePrio); + BMK_syntheticTest(cLevel, cLevelLast, compressibility, compressionParams); else - BMK_benchFileTable(fileNamesTable, nbFiles, dictFileName, cLevel, cLevelLast, compressionParams, setRealTimePrio); + BMK_benchFileTable(fileNamesTable, nbFiles, dictFileName, cLevel, cLevelLast, compressionParams); return 0; } diff --git a/programs/bench.h b/programs/bench.h index 82bb34569..d2ec94ee3 100644 --- a/programs/bench.h +++ b/programs/bench.h @@ -16,14 +16,16 @@ #define ZSTD_STATIC_LINKING_ONLY /* ZSTD_compressionParameters */ #include "zstd.h" /* ZSTD_compressionParameters */ -int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles,const char* dictFileName, - int cLevel, int cLevelLast, ZSTD_compressionParameters* compressionParams, int setRealTimePrio); +int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles, const char* dictFileName, + int cLevel, int cLevelLast, const ZSTD_compressionParameters* compressionParams); /* Set Parameters */ void BMK_setNbSeconds(unsigned nbLoops); void BMK_setBlockSize(size_t blockSize); void BMK_setNbThreads(unsigned nbThreads); +void BMK_setRealTime(unsigned priority); void BMK_setNotificationLevel(unsigned level); +void BMK_setSeparateFiles(unsigned separate); void BMK_setAdditionalParam(int additionalParam); void BMK_setDecodeOnlyMode(unsigned decodeFlag); void BMK_setLdmFlag(unsigned ldmFlag); diff --git a/programs/util.h b/programs/util.h index c5e4365d0..e44d7459d 100644 --- a/programs/util.h +++ b/programs/util.h @@ -336,7 +336,7 @@ UTIL_STATIC U64 UTIL_getFileSize(const char* infilename) } -UTIL_STATIC U64 UTIL_getTotalFileSize(const char** fileNamesTable, unsigned nbFiles) +UTIL_STATIC U64 UTIL_getTotalFileSize(const char* const * const fileNamesTable, unsigned nbFiles) { U64 total = 0; int error = 0; diff --git a/programs/zstdcli.c b/programs/zstdcli.c index 3f8367341..91af49e24 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -377,6 +377,7 @@ int main(int argCount, const char* argv[]) lastCommand = 0, nbThreads = 1, setRealTimePrio = 0, + separateFiles = 0, ldmFlag = 0; unsigned bench_nbSeconds = 3; /* would be better if this value was synchronized from bench */ size_t blockSize = 0; @@ -633,6 +634,12 @@ int main(int argCount, const char* argv[]) blockSize = readU32FromChar(&argument); break; + /* benchmark files separately (hidden option) */ + case 'S': + argument++; + separateFiles = 1; + break; + #endif /* ZSTD_NOBENCH */ /* nb of threads (hidden option) */ @@ -751,8 +758,10 @@ int main(int argCount, const char* argv[]) if (operation==zom_bench) { #ifndef ZSTD_NOBENCH BMK_setNotificationLevel(g_displayLevel); + BMK_setSeparateFiles(separateFiles); BMK_setBlockSize(blockSize); BMK_setNbThreads(nbThreads); + BMK_setRealTime(setRealTimePrio); BMK_setNbSeconds(bench_nbSeconds); BMK_setLdmFlag(ldmFlag); BMK_setLdmMinMatch(g_ldmMinMatch); @@ -763,7 +772,7 @@ int main(int argCount, const char* argv[]) if (g_ldmHashEveryLog != LDM_PARAM_DEFAULT) { BMK_setLdmHashEveryLog(g_ldmHashEveryLog); } - BMK_benchFiles(filenameTable, filenameIdx, dictFileName, cLevel, cLevelLast, &compressionParams, setRealTimePrio); + BMK_benchFiles(filenameTable, filenameIdx, dictFileName, cLevel, cLevelLast, &compressionParams); #endif (void)bench_nbSeconds; (void)blockSize; (void)setRealTimePrio; goto _end;