From 4e26bb69c78b64475ec820980d7880183585eb4a Mon Sep 17 00:00:00 2001 From: inikep Date: Mon, 14 Mar 2016 12:48:51 +0100 Subject: [PATCH 01/20] bench.c can generate a log file with average values --- programs/bench.c | 70 ++++++++++++++++++++++++++++++++++++---------- programs/bench.h | 2 +- programs/zstdcli.c | 3 ++ 3 files changed, 60 insertions(+), 15 deletions(-) diff --git a/programs/bench.c b/programs/bench.c index 1c776c4e6..811e17790 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -127,16 +127,18 @@ static U32 g_displayLevel = 2; /* 0 : no display; 1: errors; 2 : + result static int nbIterations = NBLOOPS; static size_t g_blockSize = 0; +void BMK_setNotificationLevel(unsigned level) { g_displayLevel=level; } + void BMK_SetNbIterations(int nbLoops) { nbIterations = nbLoops; - DISPLAY("- %i iterations -\n", nbIterations); + DISPLAYLEVEL(2, "- %i iterations -\n", nbIterations); } void BMK_SetBlockSize(size_t blockSize) { g_blockSize = blockSize; - DISPLAY("using blocks of size %u KB \n", (U32)(blockSize>>10)); + DISPLAYLEVEL(2, "using blocks of size %u KB \n", (U32)(blockSize>>10)); } @@ -211,13 +213,22 @@ typedef struct size_t resSize; } blockParam_t; +typedef struct +{ + float ratio; + size_t cSize; + float cSpeed; + float dSpeed; +} benchResult_t; + + #define MIN(a,b) ((a)<(b) ? (a) : (b)) #define MAX(a,b) ((a)>(b) ? (a) : (b)) static int BMK_benchMem(const void* srcBuffer, size_t srcSize, const char* displayName, int cLevel, const size_t* fileSizes, U32 nbFiles, - const void* dictBuffer, size_t dictBufferSize) + const void* dictBuffer, size_t dictBufferSize, benchResult_t *result) { const size_t blockSize = (g_blockSize ? g_blockSize : srcSize) + (!srcSize); /* avoid div by 0 */ const U32 maxNbBlocks = (U32) ((srcSize + (blockSize-1)) / blockSize) + nbFiles; @@ -232,7 +243,8 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize, ZSTD_DCtx* dctx = ZSTD_createDCtx(); U64 crcOrig = XXH64(srcBuffer, srcSize, 0); U32 nbBlocks = 0; - + size_t cSize = 0; + /* init */ if (strlen(displayName)>17) displayName += strlen(displayName)-17; /* can only display 17 characters */ @@ -270,19 +282,18 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize, /* Bench */ { int loopNb; - size_t cSize = 0; double fastestC = 100000000., fastestD = 100000000.; double ratio = 0.; U64 crcCheck = 0; - DISPLAY("\r%79s\r", ""); + DISPLAYLEVEL(2, "\r%79s\r", ""); for (loopNb = 1; loopNb <= nbIterations; loopNb++) { int nbLoops; int milliTime; U32 blockNb; /* Compression */ - DISPLAY("%2i-%-17.17s :%10u ->\r", loopNb, displayName, (U32)srcSize); + DISPLAYLEVEL(2, "%2i-%-17.17s :%10u ->\r", loopNb, displayName, (U32)srcSize); memset(compressedBuffer, 0xE5, maxCompressedSize); nbLoops = 0; @@ -307,7 +318,7 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize, cSize += blockTable[blockNb].cSize; if ((double)milliTime < fastestC*nbLoops) fastestC = (double)milliTime / nbLoops; ratio = (double)srcSize / (double)cSize; - DISPLAY("%2i-%-17.17s :%10i ->%10i (%5.3f),%6.1f MB/s\r", loopNb, displayName, (int)srcSize, (int)cSize, ratio, (double)srcSize / fastestC / 1000.); + DISPLAYLEVEL(2, "%2i-%-17.17s :%10i ->%10i (%5.3f),%6.1f MB/s\r", loopNb, displayName, (int)srcSize, (int)cSize, ratio, (double)srcSize / fastestC / 1000.); #if 1 /* Decompression */ @@ -334,7 +345,7 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize, milliTime = BMK_GetMilliSpan(milliTime); if ((double)milliTime < fastestD*nbLoops) fastestD = (double)milliTime / nbLoops; - DISPLAY("%2i-%-17.17s :%10i ->%10i (%5.3f),%6.1f MB/s ,%6.1f MB/s\r", loopNb, displayName, (int)srcSize, (int)cSize, ratio, (double)srcSize / fastestC / 1000., (double)srcSize / fastestD / 1000.); + DISPLAYLEVEL(2, "%2i-%-17.17s :%10i ->%10i (%5.3f),%6.1f MB/s ,%6.1f MB/s\r", loopNb, displayName, (int)srcSize, (int)cSize, ratio, (double)srcSize / fastestC / 1000., (double)srcSize / fastestD / 1000.); /* CRC Checking */ _findError: @@ -365,9 +376,15 @@ _findError: } if (crcOrig == crcCheck) - DISPLAY("%2i-%-17.17s :%10i ->%10i (%5.3f),%6.1f MB/s ,%6.1f MB/s \n", cLevel, displayName, (int)srcSize, (int)cSize, ratio, (double)srcSize / fastestC / 1000., (double)srcSize / fastestD / 1000.); + { + DISPLAYLEVEL(2, "%2i-%-17.17s :%10i ->%10i (%5.3f),%6.1f MB/s ,%6.1f MB/s \n", cLevel, displayName, (int)srcSize, (int)cSize, ratio, (double)srcSize / fastestC / 1000., (double)srcSize / fastestD / 1000.); + result->ratio = ratio; + result->cSize = cSize; + result->cSpeed = (double)srcSize / fastestC / 1000.; + result->dSpeed = (double)srcSize / fastestD / 1000.; + } else - DISPLAY("%2i-\n", cLevel); + DISPLAYLEVEL(2, "%2i-\n", cLevel); } /* clean up */ @@ -403,19 +420,44 @@ static void BMK_benchCLevel(void* srcBuffer, size_t benchedSize, const size_t* fileSizes, unsigned nbFiles, const void* dictBuffer, size_t dictBufferSize) { + benchResult_t result, total; + if (cLevel < 0) { int l; - for (l=1; l <= -cLevel; l++) + memset(&total, 0, sizeof(total)); + char* pch = strrchr(displayName, '\\'); + if (pch) displayName = pch+1; + + if (g_displayLevel == 1) + DISPLAY("- input %u bytes, %i iterations, %u KB blocks\n", (U32)benchedSize, nbIterations, (U32)(g_blockSize>>10)); + + for (l=1; l <= -cLevel; l++) { BMK_benchMem(srcBuffer, benchedSize, displayName, l, fileSizes, nbFiles, - dictBuffer, dictBufferSize); + dictBuffer, dictBufferSize, &result); + if (g_displayLevel == 1) { + DISPLAY("%2i:%11i (%5.3f),%6.1f MB/s,%6.1f MB/s, %s\n", -l, (int)result.cSize, result.ratio, result.cSpeed, result.dSpeed, displayName); + total.cSize += result.cSize; + total.cSpeed += result.cSpeed; + total.dSpeed += result.dSpeed; + total.ratio += result.ratio; + } + } + if (g_displayLevel == 1) + { + total.cSize /= -cLevel; + total.cSpeed /= -cLevel; + total.dSpeed /= -cLevel; + total.ratio /= -cLevel; + DISPLAY("avg%11i (%5.3f),%6.1f MB/s,%6.1f MB/s, %s\n", (int)total.cSize, total.ratio, total.cSpeed, total.dSpeed, displayName); + } return; } BMK_benchMem(srcBuffer, benchedSize, displayName, cLevel, fileSizes, nbFiles, - dictBuffer, dictBufferSize); + dictBuffer, dictBufferSize, &result); } static U64 BMK_getTotalFileSize(const char** fileNamesTable, unsigned nbFiles) diff --git a/programs/bench.h b/programs/bench.h index 9ae83690c..bf8e7d4b5 100644 --- a/programs/bench.h +++ b/programs/bench.h @@ -32,5 +32,5 @@ int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles, /* Set Parameters */ void BMK_SetNbIterations(int nbLoops); void BMK_SetBlockSize(size_t blockSize); - +void BMK_setNotificationLevel(unsigned level); diff --git a/programs/zstdcli.c b/programs/zstdcli.c index abe13013c..d710bdc43 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -291,6 +291,7 @@ int main(int argCount, const char** argv) argument++; while ((*argument >='0') && (*argument <='9')) iters *= 10, iters += *argument++ - '0'; + BMK_setNotificationLevel(displayLevel); BMK_SetNbIterations(iters); } break; @@ -305,6 +306,7 @@ int main(int argCount, const char** argv) if (*argument=='K') bSize<<=10, argument++; /* allows using KB notation */ if (*argument=='M') bSize<<=20, argument++; if (*argument=='B') argument++; + BMK_setNotificationLevel(displayLevel); BMK_SetBlockSize(bSize); } break; @@ -366,6 +368,7 @@ int main(int argCount, const char** argv) /* Check if benchmark is selected */ if (bench) { #ifndef ZSTD_NOBENCH + BMK_setNotificationLevel(displayLevel); BMK_benchFiles(filenameTable, filenameIdx, dictFileName, cLevel*rangeBench); #endif goto _end; From c034b73f8f15f6794e0ccf65d3ab4469b8d1fd65 Mon Sep 17 00:00:00 2001 From: inikep Date: Mon, 14 Mar 2016 13:13:42 +0100 Subject: [PATCH 02/20] benchResult_t changed to double --- programs/bench.c | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/programs/bench.c b/programs/bench.c index 811e17790..13ec3106a 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -215,10 +215,10 @@ typedef struct typedef struct { - float ratio; + double ratio; size_t cSize; - float cSpeed; - float dSpeed; + double cSpeed; + double dSpeed; } benchResult_t; @@ -425,7 +425,8 @@ static void BMK_benchCLevel(void* srcBuffer, size_t benchedSize, if (cLevel < 0) { int l; memset(&total, 0, sizeof(total)); - char* pch = strrchr(displayName, '\\'); + char* pch = strrchr(displayName, '\\'); /* Windows */ + if (!pch) pch = strrchr(displayName, '/'); /* Linux */ if (pch) displayName = pch+1; if (g_displayLevel == 1) @@ -437,7 +438,7 @@ static void BMK_benchCLevel(void* srcBuffer, size_t benchedSize, fileSizes, nbFiles, dictBuffer, dictBufferSize, &result); if (g_displayLevel == 1) { - DISPLAY("%2i:%11i (%5.3f),%6.1f MB/s,%6.1f MB/s, %s\n", -l, (int)result.cSize, result.ratio, result.cSpeed, result.dSpeed, displayName); + DISPLAY("%-3i%11i (%5.3f) %6.1f MB/s %6.1f MB/s %s\n", -l, (int)result.cSize, result.ratio, result.cSpeed, result.dSpeed, displayName); total.cSize += result.cSize; total.cSpeed += result.cSpeed; total.dSpeed += result.dSpeed; @@ -450,7 +451,7 @@ static void BMK_benchCLevel(void* srcBuffer, size_t benchedSize, total.cSpeed /= -cLevel; total.dSpeed /= -cLevel; total.ratio /= -cLevel; - DISPLAY("avg%11i (%5.3f),%6.1f MB/s,%6.1f MB/s, %s\n", (int)total.cSize, total.ratio, total.cSpeed, total.dSpeed, displayName); + DISPLAY("avg%11i (%5.3f) %6.1f MB/s %6.1f MB/s %s\n", (int)total.cSize, total.ratio, total.cSpeed, total.dSpeed, displayName); } return; } From 44af12deb1fd882d3651681a9a21b1356fa61ab8 Mon Sep 17 00:00:00 2001 From: inikep Date: Mon, 14 Mar 2016 15:59:04 +0100 Subject: [PATCH 03/20] removed DEFAULT_CHUNKSIZE --- programs/bench.c | 9 ++++++--- programs/fullbench.c | 1 - programs/paramgrill.c | 1 - 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/programs/bench.c b/programs/bench.c index 13ec3106a..6179dda0f 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -83,6 +83,10 @@ /* ************************************* * Constants ***************************************/ +#ifndef ZSTD_VERSION +# define ZSTD_VERSION "" +#endif + #define NBLOOPS 3 #define TIMELOOP 2500 @@ -91,7 +95,6 @@ #define GB *(1U<<30) static const size_t maxMemory = (sizeof(size_t)==4) ? (2 GB - 64 MB) : (size_t)(1ULL << ((sizeof(size_t)*8)-31)); -#define DEFAULT_CHUNKSIZE (4 MB) static U32 g_compressibilityDefault = 50; @@ -425,12 +428,12 @@ static void BMK_benchCLevel(void* srcBuffer, size_t benchedSize, if (cLevel < 0) { int l; memset(&total, 0, sizeof(total)); - char* pch = strrchr(displayName, '\\'); /* Windows */ + const char* pch = strrchr(displayName, '\\'); /* Windows */ if (!pch) pch = strrchr(displayName, '/'); /* Linux */ if (pch) displayName = pch+1; if (g_displayLevel == 1) - DISPLAY("- input %u bytes, %i iterations, %u KB blocks\n", (U32)benchedSize, nbIterations, (U32)(g_blockSize>>10)); + DISPLAY("bench %s: input %u bytes, %i iterations, %u KB blocks\n", ZSTD_VERSION, (U32)benchedSize, nbIterations, (U32)(g_blockSize>>10)); for (l=1; l <= -cLevel; l++) { BMK_benchMem(srcBuffer, benchedSize, diff --git a/programs/fullbench.c b/programs/fullbench.c index 0eeb88e85..f6a2321ec 100644 --- a/programs/fullbench.c +++ b/programs/fullbench.c @@ -94,7 +94,6 @@ #define KNUTH 2654435761U #define MAX_MEM (1984 MB) -#define DEFAULT_CHUNKSIZE (4<<20) #define COMPRESSIBILITY_DEFAULT 0.50 static const size_t sampleSize = 10000000; diff --git a/programs/paramgrill.c b/programs/paramgrill.c index fb82d926a..1b55f11c9 100644 --- a/programs/paramgrill.c +++ b/programs/paramgrill.c @@ -100,7 +100,6 @@ #define NB_LEVELS_TRACKED 30 static const size_t maxMemory = (sizeof(size_t)==4) ? (2 GB - 64 MB) : (size_t)(1ULL << ((sizeof(size_t)*8)-31)); -#define DEFAULT_CHUNKSIZE (4<<20) #define COMPRESSIBILITY_DEFAULT 0.50 static const size_t sampleSize = 10000000; From e9554b73cc6014cfb41b56170d102b11b102f13f Mon Sep 17 00:00:00 2001 From: inikep Date: Mon, 14 Mar 2016 18:10:30 +0100 Subject: [PATCH 04/20] -r# : test all compression levels from -bX to # --- programs/bench.c | 81 ++++++++++++++++++++++------------------------ programs/bench.h | 2 +- programs/zstdcli.c | 16 ++++++--- 3 files changed, 51 insertions(+), 48 deletions(-) diff --git a/programs/bench.c b/programs/bench.c index 6179dda0f..e233c17fb 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -419,49 +419,46 @@ static size_t BMK_findMaxMem(U64 requiredMem) } static void BMK_benchCLevel(void* srcBuffer, size_t benchedSize, - const char* displayName, int cLevel, + const char* displayName, int cLevel, int cLevelLast, const size_t* fileSizes, unsigned nbFiles, const void* dictBuffer, size_t dictBufferSize) { benchResult_t result, total; - - if (cLevel < 0) { - int l; - memset(&total, 0, sizeof(total)); - const char* pch = strrchr(displayName, '\\'); /* Windows */ - if (!pch) pch = strrchr(displayName, '/'); /* Linux */ - if (pch) displayName = pch+1; + int l; - if (g_displayLevel == 1) - DISPLAY("bench %s: input %u bytes, %i iterations, %u KB blocks\n", ZSTD_VERSION, (U32)benchedSize, nbIterations, (U32)(g_blockSize>>10)); + const char* pch = strrchr(displayName, '\\'); /* Windows */ + if (!pch) pch = strrchr(displayName, '/'); /* Linux */ + if (pch) displayName = pch+1; - for (l=1; l <= -cLevel; l++) { - BMK_benchMem(srcBuffer, benchedSize, - displayName, l, - fileSizes, nbFiles, - dictBuffer, dictBufferSize, &result); - if (g_displayLevel == 1) { - DISPLAY("%-3i%11i (%5.3f) %6.1f MB/s %6.1f MB/s %s\n", -l, (int)result.cSize, result.ratio, result.cSpeed, result.dSpeed, displayName); - total.cSize += result.cSize; - total.cSpeed += result.cSpeed; - total.dSpeed += result.dSpeed; - total.ratio += result.ratio; - } + memset(&result, 0, sizeof(result)); + memset(&total, 0, sizeof(total)); + + if (g_displayLevel == 1) + DISPLAY("bench %s: input %u bytes, %i iterations, %u KB blocks\n", ZSTD_VERSION, (U32)benchedSize, nbIterations, (U32)(g_blockSize>>10)); + + if (cLevelLast < cLevel) cLevelLast = cLevel; + + for (l=cLevel; l <= cLevelLast; l++) { + BMK_benchMem(srcBuffer, benchedSize, + displayName, l, + fileSizes, nbFiles, + dictBuffer, dictBufferSize, &result); + if (g_displayLevel == 1) { + DISPLAY("%-3i%11i (%5.3f) %6.1f MB/s %6.1f MB/s %s\n", -l, (int)result.cSize, result.ratio, result.cSpeed, result.dSpeed, displayName); + total.cSize += result.cSize; + total.cSpeed += result.cSpeed; + total.dSpeed += result.dSpeed; + total.ratio += result.ratio; } - if (g_displayLevel == 1) - { - total.cSize /= -cLevel; - total.cSpeed /= -cLevel; - total.dSpeed /= -cLevel; - total.ratio /= -cLevel; - DISPLAY("avg%11i (%5.3f) %6.1f MB/s %6.1f MB/s %s\n", (int)total.cSize, total.ratio, total.cSpeed, total.dSpeed, displayName); - } - return; } - BMK_benchMem(srcBuffer, benchedSize, - displayName, cLevel, - fileSizes, nbFiles, - dictBuffer, dictBufferSize, &result); + if (g_displayLevel == 1 && cLevelLast > cLevel) + { + total.cSize /= 1+cLevelLast-cLevel; + total.cSpeed /= 1+cLevelLast-cLevel; + total.dSpeed /= 1+cLevelLast-cLevel; + total.ratio /= 1+cLevelLast-cLevel; + DISPLAY("avg%11i (%5.3f) %6.1f MB/s %6.1f MB/s %s\n", (int)total.cSize, total.ratio, total.cSpeed, total.dSpeed, displayName); + } } static U64 BMK_getTotalFileSize(const char** fileNamesTable, unsigned nbFiles) @@ -497,7 +494,7 @@ static void BMK_loadFiles(void* buffer, size_t bufferSize, } static void BMK_benchFileTable(const char** fileNamesTable, unsigned nbFiles, - const char* dictFileName, int cLevel) + const char* dictFileName, int cLevel, int cLevelLast) { void* srcBuffer; size_t benchedSize; @@ -537,7 +534,7 @@ static void BMK_benchFileTable(const char** fileNamesTable, unsigned nbFiles, else displayName = fileNamesTable[0]; BMK_benchCLevel(srcBuffer, benchedSize, - displayName, cLevel, + displayName, cLevel, cLevelLast, fileSizes, nbFiles, dictBuffer, dictBufferSize); @@ -548,7 +545,7 @@ static void BMK_benchFileTable(const char** fileNamesTable, unsigned nbFiles, } -static void BMK_syntheticTest(int cLevel, double compressibility) +static void BMK_syntheticTest(int cLevel, int cLevelLast, double compressibility) { char name[20] = {0}; size_t benchedSize = 10000000; @@ -562,7 +559,7 @@ static void BMK_syntheticTest(int cLevel, double compressibility) /* Bench */ snprintf (name, sizeof(name), "Synthetic %2u%%", (unsigned)(compressibility*100)); - BMK_benchCLevel(srcBuffer, benchedSize, name, cLevel, &benchedSize, 1, NULL, 0); + BMK_benchCLevel(srcBuffer, benchedSize, name, cLevel, cLevelLast, &benchedSize, 1, NULL, 0); /* clean up */ free(srcBuffer); @@ -570,14 +567,14 @@ static void BMK_syntheticTest(int cLevel, double compressibility) int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles, - const char* dictFileName, int cLevel) + const char* dictFileName, int cLevel, int cLevelLast) { double compressibility = (double)g_compressibilityDefault / 100; if (nbFiles == 0) - BMK_syntheticTest(cLevel, compressibility); + BMK_syntheticTest(cLevel, cLevelLast, compressibility); else - BMK_benchFileTable(fileNamesTable, nbFiles, dictFileName, cLevel); + BMK_benchFileTable(fileNamesTable, nbFiles, dictFileName, cLevel, cLevelLast); return 0; } diff --git a/programs/bench.h b/programs/bench.h index bf8e7d4b5..d1d8bd88b 100644 --- a/programs/bench.h +++ b/programs/bench.h @@ -27,7 +27,7 @@ /* Main function */ int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles, - const char* dictFileName, int cLevel); + const char* dictFileName, int cLevel, int cLevelLast); /* Set Parameters */ void BMK_SetNbIterations(int nbLoops); diff --git a/programs/zstdcli.c b/programs/zstdcli.c index d710bdc43..18fabcfec 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -142,9 +142,9 @@ static int usage_advanced(const char* programName) #ifndef ZSTD_NOBENCH DISPLAY( "Benchmark arguments :\n"); DISPLAY( " -b# : benchmark file(s), using # compression level (default : 1) \n"); + DISPLAY( " -r# : test all compression levels from -bX to # (default: 1)\n"); DISPLAY( " -i# : iteration loops [1-9](default : 3)\n"); DISPLAY( " -B# : cut file into independent blocks of size # (default: no block)\n"); - DISPLAY( " -r# : test all compression levels from 1 to # (default: disabled)\n"); #endif return 0; } @@ -179,19 +179,19 @@ int main(int argCount, const char** argv) nextArgumentIsOutFileName=0, nextArgumentIsMaxDict=0; unsigned cLevel = 1; + unsigned cLevelLast = 1; const char** filenameTable = (const char**)malloc(argCount * sizeof(const char*)); /* argCount >= 1 */ unsigned filenameIdx = 0; const char* programName = argv[0]; const char* outFileName = NULL; const char* dictFileName = NULL; char* dynNameSpace = NULL; - int rangeBench = 1; unsigned maxDictSize = g_defaultMaxDictSize; unsigned dictCLevel = g_defaultDictCLevel; unsigned dictSelect = g_defaultSelectivityLevel; /* init */ - (void)rangeBench; (void)dictCLevel; /* not used when ZSTD_NOBENCH / ZSTD_NODICT set */ + (void)cLevelLast; (void)dictCLevel; /* not used when ZSTD_NOBENCH / ZSTD_NODICT set */ if (filenameTable==NULL) { DISPLAY("not enough memory\n"); exit(1); } displayOut = stderr; /* Pick out program name from path. Don't rely on stdlib because of conflicting behavior */ @@ -313,8 +313,14 @@ int main(int argCount, const char** argv) /* range bench (benchmark only) */ case 'r': - rangeBench = -1; + /* compression Level */ argument++; + if ((*argument>='0') && (*argument<='9')) { + cLevelLast = 0; + while ((*argument >= '0') && (*argument <= '9')) + cLevelLast *= 10, cLevelLast += *argument++ - '0'; + continue; + } break; #endif /* ZSTD_NOBENCH */ @@ -369,7 +375,7 @@ int main(int argCount, const char** argv) if (bench) { #ifndef ZSTD_NOBENCH BMK_setNotificationLevel(displayLevel); - BMK_benchFiles(filenameTable, filenameIdx, dictFileName, cLevel*rangeBench); + BMK_benchFiles(filenameTable, filenameIdx, dictFileName, cLevel, cLevelLast); #endif goto _end; } From 5fdd0b46d319820370bc96d0fb69212d9605cdd9 Mon Sep 17 00:00:00 2001 From: inikep Date: Mon, 14 Mar 2016 19:51:11 +0100 Subject: [PATCH 05/20] added additionalParam --- lib/zstd_internal.h | 2 ++ programs/bench.c | 26 ++++++++++++++------------ programs/bench.h | 2 +- programs/zstdcli.c | 18 ++++++++++++------ 4 files changed, 29 insertions(+), 19 deletions(-) diff --git a/lib/zstd_internal.h b/lib/zstd_internal.h index 409e5e57a..de78bb6f9 100644 --- a/lib/zstd_internal.h +++ b/lib/zstd_internal.h @@ -226,5 +226,7 @@ typedef struct { seqStore_t ZSTD_copySeqStore(const ZSTD_CCtx* ctx); +void ZSTD_setAdditionalParam(ZSTD_CCtx* ctx, int additionalParam); + #endif /* ZSTD_CCOMMON_H_MODULE */ diff --git a/programs/bench.c b/programs/bench.c index e233c17fb..94daa36cb 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -64,6 +64,7 @@ #include "mem.h" #include "zstd_static.h" +#include "zstd_internal.h" /* ZSTD_setAdditionalParam */ #include "xxhash.h" #include "datagen.h" /* RDG_genBuffer */ @@ -229,7 +230,7 @@ typedef struct #define MAX(a,b) ((a)>(b) ? (a) : (b)) static int BMK_benchMem(const void* srcBuffer, size_t srcSize, - const char* displayName, int cLevel, + const char* displayName, int cLevel, int additionalParam, const size_t* fileSizes, U32 nbFiles, const void* dictBuffer, size_t dictBufferSize, benchResult_t *result) { @@ -280,6 +281,7 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize, } } } /* warmimg up memory */ + ZSTD_setAdditionalParam(refCtx, additionalParam); RDG_genBuffer(compressedBuffer, maxCompressedSize, 0.10, 0.50, 1); /* Bench */ @@ -419,7 +421,7 @@ static size_t BMK_findMaxMem(U64 requiredMem) } static void BMK_benchCLevel(void* srcBuffer, size_t benchedSize, - const char* displayName, int cLevel, int cLevelLast, + const char* displayName, int cLevel, int cLevelLast, int additionalParam, const size_t* fileSizes, unsigned nbFiles, const void* dictBuffer, size_t dictBufferSize) { @@ -432,15 +434,15 @@ static void BMK_benchCLevel(void* srcBuffer, size_t benchedSize, memset(&result, 0, sizeof(result)); memset(&total, 0, sizeof(total)); - - if (g_displayLevel == 1) + + if (g_displayLevel == 1) DISPLAY("bench %s: input %u bytes, %i iterations, %u KB blocks\n", ZSTD_VERSION, (U32)benchedSize, nbIterations, (U32)(g_blockSize>>10)); if (cLevelLast < cLevel) cLevelLast = cLevel; for (l=cLevel; l <= cLevelLast; l++) { BMK_benchMem(srcBuffer, benchedSize, - displayName, l, + displayName, l, additionalParam, fileSizes, nbFiles, dictBuffer, dictBufferSize, &result); if (g_displayLevel == 1) { @@ -494,7 +496,7 @@ static void BMK_loadFiles(void* buffer, size_t bufferSize, } static void BMK_benchFileTable(const char** fileNamesTable, unsigned nbFiles, - const char* dictFileName, int cLevel, int cLevelLast) + const char* dictFileName, int cLevel, int cLevelLast, int additionalParam) { void* srcBuffer; size_t benchedSize; @@ -534,7 +536,7 @@ static void BMK_benchFileTable(const char** fileNamesTable, unsigned nbFiles, else displayName = fileNamesTable[0]; BMK_benchCLevel(srcBuffer, benchedSize, - displayName, cLevel, cLevelLast, + displayName, cLevel, cLevelLast, additionalParam, fileSizes, nbFiles, dictBuffer, dictBufferSize); @@ -545,7 +547,7 @@ static void BMK_benchFileTable(const char** fileNamesTable, unsigned nbFiles, } -static void BMK_syntheticTest(int cLevel, int cLevelLast, double compressibility) +static void BMK_syntheticTest(int cLevel, int cLevelLast, int additionalParam, double compressibility) { char name[20] = {0}; size_t benchedSize = 10000000; @@ -559,7 +561,7 @@ static void BMK_syntheticTest(int cLevel, int cLevelLast, double compressibility /* Bench */ snprintf (name, sizeof(name), "Synthetic %2u%%", (unsigned)(compressibility*100)); - BMK_benchCLevel(srcBuffer, benchedSize, name, cLevel, cLevelLast, &benchedSize, 1, NULL, 0); + BMK_benchCLevel(srcBuffer, benchedSize, name, cLevel, cLevelLast, additionalParam, &benchedSize, 1, NULL, 0); /* clean up */ free(srcBuffer); @@ -567,14 +569,14 @@ static void BMK_syntheticTest(int cLevel, int cLevelLast, double compressibility int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles, - const char* dictFileName, int cLevel, int cLevelLast) + const char* dictFileName, int cLevel, int cLevelLast, int additionalParam) { double compressibility = (double)g_compressibilityDefault / 100; if (nbFiles == 0) - BMK_syntheticTest(cLevel, cLevelLast, compressibility); + BMK_syntheticTest(cLevel, cLevelLast, additionalParam, compressibility); else - BMK_benchFileTable(fileNamesTable, nbFiles, dictFileName, cLevel, cLevelLast); + BMK_benchFileTable(fileNamesTable, nbFiles, dictFileName, cLevel, cLevelLast, additionalParam); return 0; } diff --git a/programs/bench.h b/programs/bench.h index d1d8bd88b..5e6d3b18d 100644 --- a/programs/bench.h +++ b/programs/bench.h @@ -27,7 +27,7 @@ /* Main function */ int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles, - const char* dictFileName, int cLevel, int cLevelLast); + const char* dictFileName, int cLevel, int cLevelLast, int additionalParam); /* Set Parameters */ void BMK_SetNbIterations(int nbLoops); diff --git a/programs/zstdcli.c b/programs/zstdcli.c index 18fabcfec..c986caee3 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -180,6 +180,7 @@ int main(int argCount, const char** argv) nextArgumentIsMaxDict=0; unsigned cLevel = 1; unsigned cLevelLast = 1; + int additionalParam = 0; const char** filenameTable = (const char**)malloc(argCount * sizeof(const char*)); /* argCount >= 1 */ unsigned filenameIdx = 0; const char* programName = argv[0]; @@ -191,7 +192,7 @@ int main(int argCount, const char** argv) unsigned dictSelect = g_defaultSelectivityLevel; /* init */ - (void)cLevelLast; (void)dictCLevel; /* not used when ZSTD_NOBENCH / ZSTD_NODICT set */ + (void)additionalParam; (void)cLevelLast; (void)dictCLevel; /* not used when ZSTD_NOBENCH / ZSTD_NODICT set */ if (filenameTable==NULL) { DISPLAY("not enough memory\n"); exit(1); } displayOut = stderr; /* Pick out program name from path. Don't rely on stdlib because of conflicting behavior */ @@ -231,7 +232,6 @@ int main(int argCount, const char** argv) argument++; while (argument[0]!=0) { - /* compression Level */ if ((*argument>='0') && (*argument<='9')) { cLevel = 0; @@ -331,9 +331,15 @@ int main(int argCount, const char** argv) dictSelect *= 10, dictSelect += *argument++ - '0'; break; - /* Pause at the end (hidden option) */ - case 'p': main_pause=1; argument++; break; - + /* Pause at the end (-p) or set an additional param (-p#) (hidden option) */ + case 'p': argument++; + if ((*argument>='0') && (*argument<='9')) { + additionalParam = 0; + while ((*argument >= '0') && (*argument <= '9')) + additionalParam *= 10, additionalParam += *argument++ - '0'; + continue; + } + main_pause=1; break; /* unknown command */ default : return badusage(programName); } @@ -375,7 +381,7 @@ int main(int argCount, const char** argv) if (bench) { #ifndef ZSTD_NOBENCH BMK_setNotificationLevel(displayLevel); - BMK_benchFiles(filenameTable, filenameIdx, dictFileName, cLevel, cLevelLast); + BMK_benchFiles(filenameTable, filenameIdx, dictFileName, cLevel, cLevelLast, additionalParam); #endif goto _end; } From d700a1a8b911aaf64cd65d714a8eda980f54765d Mon Sep 17 00:00:00 2001 From: inikep Date: Tue, 15 Mar 2016 12:18:44 +0100 Subject: [PATCH 06/20] ZSTD_setAdditionalParam --- .gitignore | 4 ---- lib/zstd_compress.c | 4 ++++ programs/.gitignore | 5 +++++ programs/bench.c | 7 +++++-- 4 files changed, 14 insertions(+), 6 deletions(-) diff --git a/.gitignore b/.gitignore index 7e142fcd1..6ca7f8a7e 100644 --- a/.gitignore +++ b/.gitignore @@ -47,7 +47,3 @@ ipch/ .directory _codelite _zstdbench - -lib/zstd_opt_LZ5.c -lib/zstd_opt_llen.c -lib/zstd_opt_nollen.c diff --git a/lib/zstd_compress.c b/lib/zstd_compress.c index fc88eb168..2ee25cc93 100644 --- a/lib/zstd_compress.c +++ b/lib/zstd_compress.c @@ -98,6 +98,7 @@ struct ZSTD_CCtx_s U32 nextToUpdate3; /* index from which to continue dictionary update */ U32 loadedDictEnd; U32 stage; + U32 additionalParam; ZSTD_parameters params; void* workSpace; size_t workSpaceSize; @@ -273,6 +274,7 @@ size_t ZSTD_copyCCtx(ZSTD_CCtx* dstCCtx, const ZSTD_CCtx* srcCCtx) dstCCtx->dictLimit = srcCCtx->dictLimit; dstCCtx->lowLimit = srcCCtx->lowLimit; dstCCtx->loadedDictEnd = srcCCtx->loadedDictEnd; + dstCCtx->additionalParam = srcCCtx->additionalParam; /* copy entropy tables */ dstCCtx->flagStaticTables = srcCCtx->flagStaticTables; @@ -2421,3 +2423,5 @@ ZSTD_parameters ZSTD_getParams(int compressionLevel, U64 srcSizeHint) return result; } + +void ZSTD_setAdditionalParam(ZSTD_CCtx* ctx, int additionalParam) { ctx->additionalParam = additionalParam; }; diff --git a/programs/.gitignore b/programs/.gitignore index 525037b94..20cab859a 100644 --- a/programs/.gitignore +++ b/programs/.gitignore @@ -34,3 +34,8 @@ paramgrill # Default result files dictionary grillResults.txt +_* + +# Misc files +*.bat +fileTests.sh diff --git a/programs/bench.c b/programs/bench.c index 94daa36cb..e0db60e5b 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -435,7 +435,7 @@ static void BMK_benchCLevel(void* srcBuffer, size_t benchedSize, memset(&result, 0, sizeof(result)); memset(&total, 0, sizeof(total)); - if (g_displayLevel == 1) + if (g_displayLevel == 1 && !additionalParam) DISPLAY("bench %s: input %u bytes, %i iterations, %u KB blocks\n", ZSTD_VERSION, (U32)benchedSize, nbIterations, (U32)(g_blockSize>>10)); if (cLevelLast < cLevel) cLevelLast = cLevel; @@ -446,7 +446,10 @@ static void BMK_benchCLevel(void* srcBuffer, size_t benchedSize, fileSizes, nbFiles, dictBuffer, dictBufferSize, &result); if (g_displayLevel == 1) { - DISPLAY("%-3i%11i (%5.3f) %6.1f MB/s %6.1f MB/s %s\n", -l, (int)result.cSize, result.ratio, result.cSpeed, result.dSpeed, displayName); + if (additionalParam) + DISPLAY("%-3i%11i (%5.3f) %6.1f MB/s %6.1f MB/s %s (p=%d)\n", -l, (int)result.cSize, result.ratio, result.cSpeed, result.dSpeed, displayName, additionalParam); + else + DISPLAY("%-3i%11i (%5.3f) %6.1f MB/s %6.1f MB/s %s\n", -l, (int)result.cSize, result.ratio, result.cSpeed, result.dSpeed, displayName); total.cSize += result.cSize; total.cSpeed += result.cSpeed; total.dSpeed += result.dSpeed; From 472638c8617fb4cb6485d09dfbce9a90bdcdc6f2 Mon Sep 17 00:00:00 2001 From: inikep Date: Wed, 23 Mar 2016 12:28:28 +0100 Subject: [PATCH 07/20] added mili_sleep and setHighPriority --- lib/zstd_compress.c | 5 ----- lib/zstd_internal.h | 2 -- programs/bench.c | 20 +++++++++++++++----- 3 files changed, 15 insertions(+), 12 deletions(-) diff --git a/lib/zstd_compress.c b/lib/zstd_compress.c index 266999ca7..28a7a1c3c 100644 --- a/lib/zstd_compress.c +++ b/lib/zstd_compress.c @@ -98,7 +98,6 @@ struct ZSTD_CCtx_s U32 nextToUpdate3; /* index from which to continue dictionary update */ U32 loadedDictEnd; U32 stage; - U32 additionalParam; ZSTD_parameters params; void* workSpace; size_t workSpaceSize; @@ -276,7 +275,6 @@ size_t ZSTD_copyCCtx(ZSTD_CCtx* dstCCtx, const ZSTD_CCtx* srcCCtx) dstCCtx->dictLimit = srcCCtx->dictLimit; dstCCtx->lowLimit = srcCCtx->lowLimit; dstCCtx->loadedDictEnd= srcCCtx->loadedDictEnd; - dstCCtx->additionalParam = srcCCtx->additionalParam; /* copy entropy tables */ dstCCtx->flagStaticTables = srcCCtx->flagStaticTables; @@ -2454,6 +2452,3 @@ ZSTD_parameters ZSTD_getParams(int compressionLevel, U64 srcSize) result.srcSize = srcSize; return result; } - - -void ZSTD_setAdditionalParam(ZSTD_CCtx* ctx, int additionalParam) { ctx->additionalParam = additionalParam; }; diff --git a/lib/zstd_internal.h b/lib/zstd_internal.h index 39258b21f..ba350c4f6 100644 --- a/lib/zstd_internal.h +++ b/lib/zstd_internal.h @@ -222,7 +222,5 @@ typedef struct { seqStore_t ZSTD_copySeqStore(const ZSTD_CCtx* ctx); -void ZSTD_setAdditionalParam(ZSTD_CCtx* ctx, int additionalParam); - #endif /* ZSTD_CCOMMON_H_MODULE */ diff --git a/programs/bench.c b/programs/bench.c index 5b2e5644a..84380400d 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -44,27 +44,34 @@ /* ************************************* * Includes ***************************************/ +#define _POSIX_C_SOURCE 199309L /* before - needed for nanosleep() */ #include /* malloc, free */ #include /* memset */ #include /* fprintf, fopen, ftello64 */ #include /* stat64 */ #include /* stat64 */ -#include /* clock_t, clock, CLOCKS_PER_SEC */ +#include /* clock_t, nanosleep, clock, CLOCKS_PER_SEC */ /* sleep : posix - windows - others */ #if !defined(_WIN32) && (defined(__unix__) || defined(__unix) || (defined(__APPLE__) && defined(__MACH__))) # include +# include /* setpriority */ # define BMK_sleep(s) sleep(s) +# define mili_sleep(mili) { struct timespec t; t.tv_sec=0; t.tv_nsec=mili*1000000L; nanosleep(&t, NULL); } +# define setHighPriority() setpriority(PRIO_PROCESS, 0, -20) #elif defined(_WIN32) # include # define BMK_sleep(s) Sleep(1000*s) +# define mili_sleep(mili) Sleep(mili) +# define setHighPriority() SetPriorityClass(GetCurrentProcess(), REALTIME_PRIORITY_CLASS) #else # define BMK_sleep(s) /* disabled */ +# define mili_sleep(mili) /* disabled */ +# define setHighPriority() /* disabled */ #endif #include "mem.h" #include "zstd_static.h" -#include "zstd_internal.h" /* ZSTD_setAdditionalParam */ #include "xxhash.h" #include "datagen.h" /* RDG_genBuffer */ @@ -199,7 +206,7 @@ typedef struct #define MAX(a,b) ((a)>(b) ? (a) : (b)) static int BMK_benchMem(const void* srcBuffer, size_t srcSize, - const char* displayName, int cLevel, int additionalParam, + const char* displayName, int cLevel, const size_t* fileSizes, U32 nbFiles, const void* dictBuffer, size_t dictBufferSize, benchResult_t *result) { @@ -247,7 +254,6 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize, } } } /* warmimg up memory */ - ZSTD_setAdditionalParam(refCtx, additionalParam); RDG_genBuffer(compressedBuffer, maxCompressedSize, 0.10, 0.50, 1); /* Bench */ @@ -275,6 +281,7 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize, DISPLAYLEVEL(2, "%2i-%-17.17s :%10u ->\r", testNb, displayName, (U32)srcSize); memset(compressedBuffer, 0xE5, maxCompressedSize); /* warm up and erase result buffer */ + mili_sleep(1); /* give processor time to other processes */ clockStart = clock(); while (clock() == clockStart); clockStart = clock(); @@ -303,6 +310,7 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize, /* Decompression */ memset(resultBuffer, 0xD6, srcSize); /* warm result buffer */ + mili_sleep(1); /* give processor time to other processes */ clockStart = clock(); while (clock() == clockStart); clockStart = clock(); @@ -403,6 +411,8 @@ static void BMK_benchCLevel(void* srcBuffer, size_t benchedSize, benchResult_t result, total; int l; + setHighPriority(); + const char* pch = strrchr(displayName, '\\'); /* Windows */ if (!pch) pch = strrchr(displayName, '/'); /* Linux */ if (pch) displayName = pch+1; @@ -417,7 +427,7 @@ static void BMK_benchCLevel(void* srcBuffer, size_t benchedSize, for (l=cLevel; l <= cLevelLast; l++) { BMK_benchMem(srcBuffer, benchedSize, - displayName, l, additionalParam, + displayName, l, fileSizes, nbFiles, dictBuffer, dictBufferSize, &result); if (g_displayLevel == 1) { From 7adceef9748ddb1e4a6b956dbb35b6fc66ddb8c9 Mon Sep 17 00:00:00 2001 From: inikep Date: Wed, 23 Mar 2016 15:53:38 +0100 Subject: [PATCH 08/20] hashLog3 added to ZSTD_CCtx --- lib/zstd_compress.c | 12 +++++++++--- lib/zstd_decompress.c | 6 +++--- lib/zstd_internal.h | 5 +---- lib/zstd_opt.h | 5 +++-- lib/zstd_static.h | 1 + programs/bench.c | 8 ++++++++ 6 files changed, 25 insertions(+), 12 deletions(-) diff --git a/lib/zstd_compress.c b/lib/zstd_compress.c index 28a7a1c3c..2e62ec2c4 100644 --- a/lib/zstd_compress.c +++ b/lib/zstd_compress.c @@ -96,6 +96,7 @@ struct ZSTD_CCtx_s U32 lowLimit; /* below that point, no more data */ U32 nextToUpdate; /* index from which to continue dictionary update */ U32 nextToUpdate3; /* index from which to continue dictionary update */ + U32 hashLog3; /* dispatch table : larger == faster, more memory */ U32 loadedDictEnd; U32 stage; ZSTD_parameters params; @@ -187,7 +188,7 @@ static size_t ZSTD_resetCCtx_advanced (ZSTD_CCtx* zc, const size_t tokenSpace = blockSize + 8*maxNbSeq; const size_t contentSize = (params.strategy == ZSTD_fast) ? 0 : (1 << params.contentLog); const size_t hSize = 1 << params.hashLog; - const size_t h3Size = (params.searchLength==3) ? (1 << HASHLOG3) : 0; + const size_t h3Size = (zc->hashLog3) ? 1 << zc->hashLog3 : 0; const size_t tableSpace = (contentSize + hSize + h3Size) * sizeof(U32); /* Check if workSpace is large enough, alloc a new one if needed */ @@ -252,12 +253,13 @@ size_t ZSTD_copyCCtx(ZSTD_CCtx* dstCCtx, const ZSTD_CCtx* srcCCtx) { if (srcCCtx->stage!=0) return ERROR(stage_wrong); + dstCCtx->hashLog3 = srcCCtx->hashLog3; /* must be before ZSTD_resetCCtx_advanced */ ZSTD_resetCCtx_advanced(dstCCtx, srcCCtx->params); /* copy tables */ { const size_t contentSize = (srcCCtx->params.strategy == ZSTD_fast) ? 0 : (1 << srcCCtx->params.contentLog); const size_t hSize = 1 << srcCCtx->params.hashLog; - const size_t h3Size = (srcCCtx->params.searchLength == 3) ? (1 << HASHLOG3) : 0; + const size_t h3Size = (srcCCtx->hashLog3) ? 1 << srcCCtx->hashLog3 : 0; const size_t tableSpace = (contentSize + hSize + h3Size) * sizeof(U32); memcpy(dstCCtx->workSpace, srcCCtx->workSpace, tableSpace); } @@ -310,7 +312,7 @@ static void ZSTD_reduceIndex (ZSTD_CCtx* zc, const U32 reducerValue) { const U32 contentSize = (zc->params.strategy == ZSTD_fast) ? 0 : (1 << zc->params.contentLog); ZSTD_reduceTable(zc->contentTable, contentSize, reducerValue); } - { const U32 h3Size = (zc->params.searchLength == 3) ? (1 << HASHLOG3) : 0; + { const U32 h3Size = (zc->hashLog3) ? 1 << zc->hashLog3 : 0; ZSTD_reduceTable(zc->hashTable3, h3Size, reducerValue); } } @@ -2185,7 +2187,11 @@ size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* zc, const void* dict, size_t dictSize, ZSTD_parameters params) { +// printf("windowLog=%d hashLog=%d\n", params.windowLog, params.hashLog); ZSTD_validateParams(¶ms); + zc->hashLog3 = (params.searchLength==3) ? ZSTD_HASHLOG3 : 0; +// if (zc->hashLog3 > params.windowLog) zc->hashLog3 = params.windowLog; +// printf("windowLog=%d hashLog=%d hashLog3=%d \n", params.windowLog, params.hashLog, zc->hashLog3); { size_t const errorCode = ZSTD_resetCCtx_advanced(zc, params); if (ZSTD_isError(errorCode)) return errorCode; } diff --git a/lib/zstd_decompress.c b/lib/zstd_decompress.c index c90b88552..ef99175e8 100644 --- a/lib/zstd_decompress.c +++ b/lib/zstd_decompress.c @@ -149,7 +149,7 @@ size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx) dctx->hufTableX4[0] = HufLog; dctx->flagStaticTables = 0; dctx->fParams.mml = MINMATCH; /* overwritten by frame but forces ZSTD_btopt to MINMATCH in block mode */ - ZSTD_LOG_BLOCK("%p: ZSTD_decompressBegin searchLength=%d\n", dctx->base, dctx->params.searchLength); + ZSTD_LOG_BLOCK("%p: ZSTD_decompressBegin searchLength=%d\n", dctx->base, dctx->fParams.mml); return 0; } @@ -845,7 +845,7 @@ static size_t ZSTD_decompressBlock_internal(ZSTD_DCtx* dctx, if (srcSize >= ZSTD_BLOCKSIZE_MAX) return ERROR(srcSize_wrong); - ZSTD_LOG_BLOCK("%p: ZSTD_decompressBlock_internal searchLength=%d\n", dctx->base, dctx->params.searchLength); + ZSTD_LOG_BLOCK("%p: ZSTD_decompressBlock_internal searchLength=%d\n", dctx->base, dctx->fParams.mml); /* Decode literals sub-block */ litCSize = ZSTD_decodeLiteralsBlock(dctx, src, srcSize); @@ -953,7 +953,7 @@ size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx, const void* dict, size_t dictSize) { ZSTD_decompressBegin_usingDict(dctx, dict, dictSize); - ZSTD_LOG_BLOCK("%p: ZSTD_decompressBegin_usingDict searchLength=%d\n", dctx->base, dctx->params.searchLength); + ZSTD_LOG_BLOCK("%p: ZSTD_decompressBegin_usingDict searchLength=%d\n", dctx->base, dctx->fParams.mml); ZSTD_checkContinuity(dctx, dst); return ZSTD_decompressFrame(dctx, dst, dstCapacity, src, srcSize); } diff --git a/lib/zstd_internal.h b/lib/zstd_internal.h index ba350c4f6..3b68654e3 100644 --- a/lib/zstd_internal.h +++ b/lib/zstd_internal.h @@ -51,9 +51,7 @@ * Common constants ***************************************/ #define ZSTD_OPT_DEBUG 0 // 1 = tableID=0; 3 = price func tests; 5 = check encoded sequences; 9 = full logs -#if defined(ZSTD_OPT_DEBUG) && ZSTD_OPT_DEBUG>0 - #include -#endif +#include #if defined(ZSTD_OPT_DEBUG) && ZSTD_OPT_DEBUG>=9 #define ZSTD_LOG_PARSER(...) printf(__VA_ARGS__) #define ZSTD_LOG_ENCODE(...) printf(__VA_ARGS__) @@ -99,7 +97,6 @@ typedef enum { bt_compressed, bt_raw, bt_rle, bt_end } blockType_t; #define MINMATCH 4 #define REPCODE_STARTVALUE 1 -#define HASHLOG3 17 #define Litbits 8 #define MLbits 7 diff --git a/lib/zstd_opt.h b/lib/zstd_opt.h index b8c9d67a5..e20379c11 100644 --- a/lib/zstd_opt.h +++ b/lib/zstd_opt.h @@ -196,17 +196,18 @@ MEM_STATIC void ZSTD_updatePrice(seqStore_t* seqStorePtr, U32 litLength, const B static U32 ZSTD_insertAndFindFirstIndexHash3 (ZSTD_CCtx* zc, const BYTE* ip) { U32* const hashTable3 = zc->hashTable3; + U32 const hashLog3 = zc->hashLog3; const BYTE* const base = zc->base; const U32 target = (U32)(ip - base); U32 idx = zc->nextToUpdate3; while(idx < target) { - hashTable3[ZSTD_hash3Ptr(base+idx, HASHLOG3)] = idx; + hashTable3[ZSTD_hash3Ptr(base+idx, hashLog3)] = idx; idx++; } zc->nextToUpdate3 = target; - return hashTable3[ZSTD_hash3Ptr(ip, HASHLOG3)]; + return hashTable3[ZSTD_hash3Ptr(ip, hashLog3)]; } diff --git a/lib/zstd_static.h b/lib/zstd_static.h index 4ae771fde..d41bc841f 100644 --- a/lib/zstd_static.h +++ b/lib/zstd_static.h @@ -63,6 +63,7 @@ extern "C" { #define ZSTD_CONTENTLOG_MIN 4 #define ZSTD_HASHLOG_MAX 28 #define ZSTD_HASHLOG_MIN 12 +#define ZSTD_HASHLOG3 17 #define ZSTD_SEARCHLOG_MAX (ZSTD_CONTENTLOG_MAX-1) #define ZSTD_SEARCHLOG_MIN 1 #define ZSTD_SEARCHLENGTH_MAX 7 diff --git a/programs/bench.c b/programs/bench.c index 84380400d..d3e9c1f1d 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -288,7 +288,15 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize, for (nbLoops = 0 ; BMK_clockSpan(clockStart) < clockLoop ; nbLoops++) { U32 blockNb; +#if 0 ZSTD_compressBegin_usingDict(refCtx, dictBuffer, dictBufferSize, cLevel); +#else + ZSTD_parameters params = ZSTD_getParams(cLevel, dictBufferSize ? dictBufferSize : blockSize); + // printf("cLevel=%d dictBufferSize=%d srcSize=%d params.srcSize=%d \n", cLevel, (int)dictBufferSize, (int)blockTable[0].srcSize, (int)params.srcSize); + params.srcSize = 0; + ZSTD_compressBegin_advanced(refCtx, dictBuffer, dictBufferSize, params); +#endif + for (blockNb=0; blockNb Date: Wed, 23 Mar 2016 20:30:26 +0100 Subject: [PATCH 09/20] added ZSTD_compressBegin_targetSrcSize --- lib/zstd_compress.c | 21 ++++++++++++++++++--- lib/zstd_internal.h | 2 +- lib/zstd_static.h | 3 ++- programs/bench.c | 38 +++++++++++++++++--------------------- programs/bench.h | 3 ++- programs/zstdcli.c | 14 +++++++------- 6 files changed, 47 insertions(+), 34 deletions(-) diff --git a/lib/zstd_compress.c b/lib/zstd_compress.c index 2e62ec2c4..43e6e663a 100644 --- a/lib/zstd_compress.c +++ b/lib/zstd_compress.c @@ -97,6 +97,7 @@ struct ZSTD_CCtx_s U32 nextToUpdate; /* index from which to continue dictionary update */ U32 nextToUpdate3; /* index from which to continue dictionary update */ U32 hashLog3; /* dispatch table : larger == faster, more memory */ + U32 targetSrcSize; /* optimize compression for this source size */ U32 loadedDictEnd; U32 stage; ZSTD_parameters params; @@ -240,6 +241,7 @@ static size_t ZSTD_resetCCtx_advanced (ZSTD_CCtx* zc, zc->hbSize = 0; zc->stage = 0; zc->loadedDictEnd = 0; + zc->targetSrcSize = 0; return 0; } @@ -277,6 +279,7 @@ size_t ZSTD_copyCCtx(ZSTD_CCtx* dstCCtx, const ZSTD_CCtx* srcCCtx) dstCCtx->dictLimit = srcCCtx->dictLimit; dstCCtx->lowLimit = srcCCtx->lowLimit; dstCCtx->loadedDictEnd= srcCCtx->loadedDictEnd; + dstCCtx->targetSrcSize= srcCCtx->targetSrcSize; /* copy entropy tables */ dstCCtx->flagStaticTables = srcCCtx->flagStaticTables; @@ -2180,6 +2183,7 @@ static size_t ZSTD_compress_insertDictionary(ZSTD_CCtx* zc, const void* dict, si } } +extern int g_additionalParam; /*! ZSTD_compressBegin_advanced() : * @return : 0, or an error code */ @@ -2187,10 +2191,10 @@ size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* zc, const void* dict, size_t dictSize, ZSTD_parameters params) { -// printf("windowLog=%d hashLog=%d\n", params.windowLog, params.hashLog); +// printf("windowLog=%d hashLog=%d targetSrcSize=%d\n", params.windowLog, params.hashLog, zc->targetSrcSize); ZSTD_validateParams(¶ms); - zc->hashLog3 = (params.searchLength==3) ? ZSTD_HASHLOG3 : 0; -// if (zc->hashLog3 > params.windowLog) zc->hashLog3 = params.windowLog; + U32 hashLog3 = (!zc->targetSrcSize || zc->targetSrcSize >= 8192) ? ZSTD_HASHLOG3_MAX : ((zc->targetSrcSize >= 2048) ? ZSTD_HASHLOG3_MIN + 1 : ZSTD_HASHLOG3_MIN); + zc->hashLog3 = (params.searchLength==3) ? hashLog3 : 0; // printf("windowLog=%d hashLog=%d hashLog3=%d \n", params.windowLog, params.hashLog, zc->hashLog3); { size_t const errorCode = ZSTD_resetCCtx_advanced(zc, params); @@ -2220,6 +2224,15 @@ size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* zc, } +size_t ZSTD_compressBegin_targetSrcSize(ZSTD_CCtx* zc, const void* dict, size_t dictSize, size_t targetSrcSize, int compressionLevel) +{ + zc->targetSrcSize = dictSize ? dictSize : targetSrcSize; + ZSTD_parameters params = ZSTD_getParams(compressionLevel, zc->targetSrcSize); + params.srcSize = 0; + ZSTD_LOG_BLOCK("%p: ZSTD_compressBegin_targetSrcSize compressionLevel=%d\n", zc->base, compressionLevel); + return ZSTD_compressBegin_advanced(zc, dict, dictSize, params); +} + size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* zc, const void* dict, size_t dictSize, int compressionLevel) { ZSTD_parameters params = ZSTD_getParams(compressionLevel, dictSize); @@ -2309,12 +2322,14 @@ size_t ZSTD_compress_advanced (ZSTD_CCtx* ctx, size_t ZSTD_compress_usingDict(ZSTD_CCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, const void* dict, size_t dictSize, int compressionLevel) { ZSTD_LOG_BLOCK("%p: ZSTD_compress_usingDict srcSize=%d dictSize=%d compressionLevel=%d\n", ctx->base, (int)srcSize, (int)dictSize, compressionLevel); + ctx->targetSrcSize = srcSize; return ZSTD_compress_advanced(ctx, dst, dstCapacity, src, srcSize, dict, dictSize, ZSTD_getParams(compressionLevel, srcSize)); } size_t ZSTD_compressCCtx (ZSTD_CCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, int compressionLevel) { ZSTD_LOG_BLOCK("%p: ZSTD_compressCCtx srcSize=%d compressionLevel=%d\n", ctx->base, (int)srcSize, compressionLevel); + ctx->targetSrcSize = srcSize; return ZSTD_compress_advanced(ctx, dst, dstCapacity, src, srcSize, NULL, 0, ZSTD_getParams(compressionLevel, srcSize)); } diff --git a/lib/zstd_internal.h b/lib/zstd_internal.h index 3b68654e3..1358eb8ab 100644 --- a/lib/zstd_internal.h +++ b/lib/zstd_internal.h @@ -218,6 +218,6 @@ typedef struct { } seqStore_t; seqStore_t ZSTD_copySeqStore(const ZSTD_CCtx* ctx); - +size_t ZSTD_compressBegin_targetSrcSize(ZSTD_CCtx* zc, const void* dict, size_t dictSize, size_t targetSrcSize, int compressionLevel); #endif /* ZSTD_CCOMMON_H_MODULE */ diff --git a/lib/zstd_static.h b/lib/zstd_static.h index d41bc841f..d88fbce82 100644 --- a/lib/zstd_static.h +++ b/lib/zstd_static.h @@ -63,7 +63,8 @@ extern "C" { #define ZSTD_CONTENTLOG_MIN 4 #define ZSTD_HASHLOG_MAX 28 #define ZSTD_HASHLOG_MIN 12 -#define ZSTD_HASHLOG3 17 +#define ZSTD_HASHLOG3_MAX 17 +#define ZSTD_HASHLOG3_MIN 15 #define ZSTD_SEARCHLOG_MAX (ZSTD_CONTENTLOG_MAX-1) #define ZSTD_SEARCHLOG_MIN 1 #define ZSTD_SEARCHLENGTH_MAX 7 diff --git a/programs/bench.c b/programs/bench.c index d3e9c1f1d..1468bcb2b 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -72,8 +72,9 @@ #include "mem.h" #include "zstd_static.h" +#include "zstd_internal.h" /* ZSTD_compressBegin_targetSrcSize */ +#include "datagen.h" /* RDG_genBuffer */ #include "xxhash.h" -#include "datagen.h" /* RDG_genBuffer */ /* ************************************* @@ -139,9 +140,12 @@ static U32 g_displayLevel = 2; /* 0 : no display; 1: errors; 2 : + result ***************************************/ static U32 g_nbIterations = NBLOOPS; static size_t g_blockSize = 0; +int g_additionalParam = 0; void BMK_setNotificationLevel(unsigned level) { g_displayLevel=level; } +void BMK_setAdditionalParam(int additionalParam) { g_additionalParam=additionalParam; } + void BMK_SetNbIterations(unsigned nbLoops) { g_nbIterations = nbLoops; @@ -288,15 +292,7 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize, for (nbLoops = 0 ; BMK_clockSpan(clockStart) < clockLoop ; nbLoops++) { U32 blockNb; -#if 0 - ZSTD_compressBegin_usingDict(refCtx, dictBuffer, dictBufferSize, cLevel); -#else - ZSTD_parameters params = ZSTD_getParams(cLevel, dictBufferSize ? dictBufferSize : blockSize); - // printf("cLevel=%d dictBufferSize=%d srcSize=%d params.srcSize=%d \n", cLevel, (int)dictBufferSize, (int)blockTable[0].srcSize, (int)params.srcSize); - params.srcSize = 0; - ZSTD_compressBegin_advanced(refCtx, dictBuffer, dictBufferSize, params); -#endif - + ZSTD_compressBegin_targetSrcSize(refCtx, dictBuffer, dictBufferSize, blockSize, cLevel); for (blockNb=0; blockNb>10)); if (cLevelLast < cLevel) cLevelLast = cLevel; @@ -439,8 +435,8 @@ static void BMK_benchCLevel(void* srcBuffer, size_t benchedSize, fileSizes, nbFiles, dictBuffer, dictBufferSize, &result); if (g_displayLevel == 1) { - if (additionalParam) - DISPLAY("%-3i%11i (%5.3f) %6.1f MB/s %6.1f MB/s %s (p=%d)\n", -l, (int)result.cSize, result.ratio, result.cSpeed, result.dSpeed, displayName, additionalParam); + if (g_additionalParam) + DISPLAY("%-3i%11i (%5.3f) %6.1f MB/s %6.1f MB/s %s (param=%d)\n", -l, (int)result.cSize, result.ratio, result.cSpeed, result.dSpeed, displayName, g_additionalParam); else DISPLAY("%-3i%11i (%5.3f) %6.1f MB/s %6.1f MB/s %s\n", -l, (int)result.cSize, result.ratio, result.cSpeed, result.dSpeed, displayName); total.cSize += result.cSize; @@ -491,7 +487,7 @@ static void BMK_loadFiles(void* buffer, size_t bufferSize, } static void BMK_benchFileTable(const char** fileNamesTable, unsigned nbFiles, - const char* dictFileName, int cLevel, int cLevelLast, int additionalParam) + const char* dictFileName, int cLevel, int cLevelLast) { void* srcBuffer; size_t benchedSize; @@ -531,7 +527,7 @@ static void BMK_benchFileTable(const char** fileNamesTable, unsigned nbFiles, else displayName = fileNamesTable[0]; BMK_benchCLevel(srcBuffer, benchedSize, - displayName, cLevel, cLevelLast, additionalParam, + displayName, cLevel, cLevelLast, fileSizes, nbFiles, dictBuffer, dictBufferSize); @@ -542,7 +538,7 @@ static void BMK_benchFileTable(const char** fileNamesTable, unsigned nbFiles, } -static void BMK_syntheticTest(int cLevel, int cLevelLast, int additionalParam, double compressibility) +static void BMK_syntheticTest(int cLevel, int cLevelLast, double compressibility) { char name[20] = {0}; size_t benchedSize = 10000000; @@ -556,7 +552,7 @@ static void BMK_syntheticTest(int cLevel, int cLevelLast, int additionalParam, d /* Bench */ snprintf (name, sizeof(name), "Synthetic %2u%%", (unsigned)(compressibility*100)); - BMK_benchCLevel(srcBuffer, benchedSize, name, cLevel, cLevelLast, additionalParam, &benchedSize, 1, NULL, 0); + BMK_benchCLevel(srcBuffer, benchedSize, name, cLevel, cLevelLast, &benchedSize, 1, NULL, 0); /* clean up */ free(srcBuffer); @@ -564,14 +560,14 @@ static void BMK_syntheticTest(int cLevel, int cLevelLast, int additionalParam, d int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles, - const char* dictFileName, int cLevel, int cLevelLast, int additionalParam) + const char* dictFileName, int cLevel, int cLevelLast) { double const compressibility = (double)g_compressibilityDefault / 100; if (nbFiles == 0) - BMK_syntheticTest(cLevel, cLevelLast, additionalParam, compressibility); + BMK_syntheticTest(cLevel, cLevelLast, compressibility); else - BMK_benchFileTable(fileNamesTable, nbFiles, dictFileName, cLevel, cLevelLast, additionalParam); + BMK_benchFileTable(fileNamesTable, nbFiles, dictFileName, cLevel, cLevelLast); return 0; } diff --git a/programs/bench.h b/programs/bench.h index bc5ffa423..3a1ca3a20 100644 --- a/programs/bench.h +++ b/programs/bench.h @@ -27,10 +27,11 @@ /* Main function */ int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles, - const char* dictFileName, int cLevel, int cLevelLast, int additionalParam); + const char* dictFileName, int cLevel, int cLevelLast); /* Set Parameters */ void BMK_SetNbIterations(unsigned nbLoops); void BMK_SetBlockSize(size_t blockSize); +void BMK_setAdditionalParam(int additionalParam); void BMK_setNotificationLevel(unsigned level); diff --git a/programs/zstdcli.c b/programs/zstdcli.c index a14fe26de..0c060bdbe 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -183,7 +183,6 @@ int main(int argCount, const char** argv) nextArgumentIsMaxDict=0; unsigned cLevel = 1; unsigned cLevelLast = 1; - int additionalParam = 0; const char** filenameTable = (const char**)malloc(argCount * sizeof(const char*)); /* argCount >= 1 */ unsigned filenameIdx = 0; const char* programName = argv[0]; @@ -195,7 +194,7 @@ int main(int argCount, const char** argv) unsigned dictSelect = g_defaultSelectivityLevel; /* init */ - (void)additionalParam; (void)cLevelLast; (void)dictCLevel; /* not used when ZSTD_NOBENCH / ZSTD_NODICT set */ + (void)cLevelLast; (void)dictCLevel; /* not used when ZSTD_NOBENCH / ZSTD_NODICT set */ if (filenameTable==NULL) { DISPLAY("not enough memory\n"); exit(1); } displayOut = stderr; /* Pick out program name from path. Don't rely on stdlib because of conflicting behavior */ @@ -321,7 +320,6 @@ int main(int argCount, const char** argv) cLevelLast = 0; while ((*argument >= '0') && (*argument <= '9')) cLevelLast *= 10, cLevelLast += *argument++ - '0'; - continue; } break; #endif /* ZSTD_NOBENCH */ @@ -336,12 +334,14 @@ int main(int argCount, const char** argv) /* Pause at the end (-p) or set an additional param (-p#) (hidden option) */ case 'p': argument++; if ((*argument>='0') && (*argument<='9')) { - additionalParam = 0; + int additionalParam = 0; while ((*argument >= '0') && (*argument <= '9')) additionalParam *= 10, additionalParam += *argument++ - '0'; - continue; + BMK_setAdditionalParam(additionalParam); + } else { + main_pause=1; } - main_pause=1; break; + break; /* unknown command */ default : CLEAN_RETURN(badusage(programName)); } @@ -383,7 +383,7 @@ int main(int argCount, const char** argv) if (bench) { #ifndef ZSTD_NOBENCH BMK_setNotificationLevel(displayLevel); - BMK_benchFiles(filenameTable, filenameIdx, dictFileName, cLevel, cLevelLast, additionalParam); + BMK_benchFiles(filenameTable, filenameIdx, dictFileName, cLevel, cLevelLast); #endif goto _end; } From 5cc4efdaf8531399242d37d8a2ebc3df1e9364f6 Mon Sep 17 00:00:00 2001 From: inikep Date: Fri, 25 Mar 2016 10:52:25 +0100 Subject: [PATCH 10/20] created zstd_stats.h --- lib/zstd_compress.c | 32 +++------ lib/zstd_internal.h | 20 +++--- lib/zstd_opt.h | 4 +- lib/zstd_stats.h | 164 ++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 188 insertions(+), 32 deletions(-) create mode 100644 lib/zstd_stats.h diff --git a/lib/zstd_compress.c b/lib/zstd_compress.c index 43e6e663a..ddb816b18 100644 --- a/lib/zstd_compress.c +++ b/lib/zstd_compress.c @@ -97,11 +97,11 @@ struct ZSTD_CCtx_s U32 nextToUpdate; /* index from which to continue dictionary update */ U32 nextToUpdate3; /* index from which to continue dictionary update */ U32 hashLog3; /* dispatch table : larger == faster, more memory */ - U32 targetSrcSize; /* optimize compression for this source size */ U32 loadedDictEnd; U32 stage; ZSTD_parameters params; void* workSpace; + size_t targetSrcSize; /* optimize compression for this source size */ size_t workSpaceSize; size_t blockSize; size_t hbSize; @@ -781,12 +781,7 @@ MEM_STATIC void ZSTD_storeSeq(seqStore_t* seqStorePtr, size_t litLength, const B printf("pos %6u : %3u literals & match %3u bytes at distance %6u \n", (U32)(literals - g_start), (U32)litLength, (U32)matchCode+MINMATCH, (U32)offsetCode); #endif -#if ZSTD_OPT_DEBUG == 3 - if (offsetCode == 0) seqStorePtr->realRepSum++; - seqStorePtr->realSeqSum++; - seqStorePtr->realMatchSum += matchCode; - seqStorePtr->realLitSum += litLength; -#endif + ZSTD_statsUpdatePrices(&seqStorePtr->stats, litLength, literals, offsetCode, matchCode); /* copy Literals */ ZSTD_wildcopy(seqStorePtr->lit, literals, litLength); @@ -1696,6 +1691,7 @@ _storeSequence: { size_t const lastLLSize = iend - anchor; memcpy(seqStorePtr->lit, anchor, lastLLSize); seqStorePtr->lit += lastLLSize; + ZSTD_statsUpdatePrices(&seqStorePtr->stats, lastLLSize, anchor, 0, 0); } } @@ -1942,6 +1938,8 @@ static size_t ZSTD_compressBlock_internal(ZSTD_CCtx* zc, void* dst, size_t dstCa } + + static size_t ZSTD_compress_generic (ZSTD_CCtx* zc, void* dst, size_t dstCapacity, const void* src, size_t srcSize) @@ -1952,15 +1950,13 @@ static size_t ZSTD_compress_generic (ZSTD_CCtx* zc, BYTE* const ostart = (BYTE*)dst; BYTE* op = ostart; const U32 maxDist = 1 << zc->params.windowLog; -#if ZSTD_OPT_DEBUG == 3 - seqStore_t* ssPtr = &zc->seqStore; - static U32 priceFunc = 0; - ssPtr->realMatchSum = ssPtr->realLitSum = ssPtr->realSeqSum = ssPtr->realRepSum = 1; - ssPtr->priceFunc = priceFunc; -#endif + ZSTD_stats_t* stats = &zc->seqStore.stats; + + ZSTD_statsInit(stats); while (remaining) { size_t cSize; + ZSTD_statsResetFreqs(stats); if (dstCapacity < ZSTD_blockHeaderSize + MIN_CBLOCK_SIZE) return ERROR(dstSize_tooSmall); /* not enough space to store compressed block */ if (remaining < blockSize) blockSize = remaining; @@ -1992,12 +1988,7 @@ static size_t ZSTD_compress_generic (ZSTD_CCtx* zc, op += cSize; } -#if ZSTD_OPT_DEBUG == 3 - ssPtr->realMatchSum += ssPtr->realSeqSum * ((zc->params.searchLength == 3) ? 3 : 4); - printf("avgMatchL=%.2f avgLitL=%.2f match=%.1f%% lit=%.1f%% reps=%d seq=%d priceFunc=%d\n", (float)ssPtr->realMatchSum/ssPtr->realSeqSum, (float)ssPtr->realLitSum/ssPtr->realSeqSum, 100.0*ssPtr->realMatchSum/(ssPtr->realMatchSum+ssPtr->realLitSum), 100.0*ssPtr->realLitSum/(ssPtr->realMatchSum+ssPtr->realLitSum), ssPtr->realRepSum, ssPtr->realSeqSum, ssPtr->priceFunc); - priceFunc++; -#endif - + ZSTD_statsPrint(stats, zc->params.searchLength); return op-ostart; } @@ -2466,9 +2457,6 @@ ZSTD_parameters ZSTD_getParams(int compressionLevel, U64 srcSize) int tableID = ((srcSize-1) <= 256 KB) + ((srcSize-1) <= 128 KB) + ((srcSize-1) <= 16 KB); /* intentional underflow for srcSizeHint == 0 */ if (compressionLevel<=0) compressionLevel = 1; if (compressionLevel > ZSTD_MAX_CLEVEL) compressionLevel = ZSTD_MAX_CLEVEL; -#if ZSTD_OPT_DEBUG >= 1 - tableID=0; -#endif result = ZSTD_defaultParameters[tableID][compressionLevel]; result.srcSize = srcSize; return result; diff --git a/lib/zstd_internal.h b/lib/zstd_internal.h index 1358eb8ab..3561291f2 100644 --- a/lib/zstd_internal.h +++ b/lib/zstd_internal.h @@ -50,7 +50,7 @@ /*-************************************* * Common constants ***************************************/ -#define ZSTD_OPT_DEBUG 0 // 1 = tableID=0; 3 = price func tests; 5 = check encoded sequences; 9 = full logs +#define ZSTD_OPT_DEBUG 0 // 3 = compression stats; 5 = check encoded sequences; 9 = full logs #include #if defined(ZSTD_OPT_DEBUG) && ZSTD_OPT_DEBUG>=9 #define ZSTD_LOG_PARSER(...) printf(__VA_ARGS__) @@ -176,6 +176,16 @@ typedef struct { U32 rep2; } ZSTD_optimal_t; +#if ZSTD_OPT_DEBUG == 3 + #include "zstd_stats.h" +#else + typedef struct { U32 unused; } ZSTD_stats_t; + MEM_STATIC void ZSTD_statsPrint(ZSTD_stats_t* stats, U32 searchLength) { (void)stats; (void)searchLength; }; + MEM_STATIC void ZSTD_statsInit(ZSTD_stats_t* stats) { (void)stats; }; + MEM_STATIC void ZSTD_statsResetFreqs(ZSTD_stats_t* stats) { (void)stats; }; + MEM_STATIC void ZSTD_statsUpdatePrices(ZSTD_stats_t* stats, U32 litLength, const BYTE* literals, U32 offset, U32 matchLength) { (void)stats; (void)litLength; (void)literals; (void)offset; (void)matchLength; }; +#endif + typedef struct { void* buffer; U32* offsetStart; @@ -208,13 +218,7 @@ typedef struct { U32 log2litSum; U32 log2offCodeSum; U32 factor; -#if ZSTD_OPT_DEBUG == 3 - U32 realMatchSum; - U32 realLitSum; - U32 realSeqSum; - U32 realRepSum; - U32 priceFunc; -#endif + ZSTD_stats_t stats; } seqStore_t; seqStore_t ZSTD_copySeqStore(const ZSTD_CCtx* ctx); diff --git a/lib/zstd_opt.h b/lib/zstd_opt.h index e20379c11..bbfb11cab 100644 --- a/lib/zstd_opt.h +++ b/lib/zstd_opt.h @@ -130,8 +130,8 @@ FORCE_INLINE U32 ZSTD_getPrice(seqStore_t* seqStorePtr, U32 litLength, const BYT if (matchLength >= MaxML) matchLength = MaxML; price += ZSTD_getLiteralPrice(seqStorePtr, litLength, literals) + seqStorePtr->log2matchLengthSum - ZSTD_highbit(seqStorePtr->matchLengthFreq[matchLength]+1); -#if ZSTD_OPT_DEBUG == 3 - switch (seqStorePtr->priceFunc) { +#if ZSTD_OPT_DEBUG == 333 + switch (seqStorePtr->) { default: case 0: return 1 + price + ((seqStorePtr->litSum>>5) / seqStorePtr->litLengthSum) + ((seqStorePtr->litSum<<1) / (seqStorePtr->litSum + seqStorePtr->matchSum)); diff --git a/lib/zstd_stats.h b/lib/zstd_stats.h new file mode 100644 index 000000000..8d70191c4 --- /dev/null +++ b/lib/zstd_stats.h @@ -0,0 +1,164 @@ +/* + zstd - standard compression library + Header File for static linking only + Copyright (C) 2014-2016, Yann Collet. + + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - zstd homepage : http://www.zstd.net +*/ +#ifndef ZSTD_STATS_H +#define ZSTD_STATS_H + + +#if defined (__cplusplus) +extern "C" { +#endif + +/*-************************************* +* Dependencies +***************************************/ +//#include "zstd.h" +//#include "mem.h" + + +/*-************************************* +* Constants +***************************************/ +//#define ZSTD_MAGICNUMBER 0xFD2FB526 /* v0.6 */ + + +/*-************************************* +* Types +***************************************/ +typedef struct { + U32 priceOffset, priceOffCode, priceMatchLength, priceLiteral, priceLitLength, priceDumpsLength; + U32 totalMatchSum, totalLitSum, totalSeqSum, totalRepSum; + U32 litSum, matchLengthSum, litLengthSum, offCodeSum; + U32 matchLengthFreq[1<totalMatchSum += stats->totalSeqSum * ((searchLength == 3) ? 3 : 4); + printf("avgMatchL=%.2f avgLitL=%.2f match=%.1f%% lit=%.1f%% reps=%d seq=%d\n", (float)stats->totalMatchSum/stats->totalSeqSum, (float)stats->totalLitSum/stats->totalSeqSum, 100.0*stats->totalMatchSum/(stats->totalMatchSum+stats->totalLitSum), 100.0*stats->totalLitSum/(stats->totalMatchSum+stats->totalLitSum), stats->totalRepSum, stats->totalSeqSum); + printf("SumBytes=%d Offset=%d OffCode=%d Match=%d Literal=%d LitLength=%d DumpsLength=%d\n", (stats->priceOffset+stats->priceOffCode+stats->priceMatchLength+stats->priceLiteral+stats->priceLitLength+stats->priceDumpsLength)/8, stats->priceOffset/8, stats->priceOffCode/8, stats->priceMatchLength/8, stats->priceLiteral/8, stats->priceLitLength/8, stats->priceDumpsLength/8); +} + +MEM_STATIC void ZSTD_statsInit(ZSTD_stats_t* stats) +{ + stats->totalLitSum = stats->totalMatchSum = stats->totalSeqSum = stats->totalRepSum = 1; + stats->priceOffset = stats->priceOffCode = stats->priceMatchLength = stats->priceLiteral = stats->priceLitLength = stats->priceDumpsLength = 0; +} + +MEM_STATIC void ZSTD_statsResetFreqs(ZSTD_stats_t* stats) +{ + unsigned u; + + stats->litSum = (1<litLengthSum = (1<matchLengthSum = (1<offCodeSum = (1<litFreq[u] = 1; + for (u=0; u<=MaxLL; u++) + stats->litLengthFreq[u] = 1; + for (u=0; u<=MaxML; u++) + stats->matchLengthFreq[u] = 1; + for (u=0; u<=MaxOff; u++) + stats->offCodeFreq[u] = 1; +} + +MEM_STATIC void ZSTD_statsUpdatePrices(ZSTD_stats_t* stats, U32 litLength, const BYTE* literals, U32 offset, U32 matchLength) +{ + /* offset */ + BYTE offCode = offset ? (BYTE)ZSTD_highbit(offset+1) + 1 : 0; + stats->priceOffCode += ZSTD_highbit(stats->offCodeSum+1) - ZSTD_highbit(stats->offCodeFreq[offCode]+1); + stats->priceOffset += (offCode-1) + (!offCode); + + /* match Length */ + stats->priceDumpsLength += ((matchLength >= MaxML)<<3) + ((matchLength >= 255+MaxML)<<4) + ((matchLength>=(1<<15))<<3); + stats->priceMatchLength += ZSTD_highbit(stats->matchLengthSum+1) - ZSTD_highbit(stats->matchLengthFreq[(matchLength >= MaxML) ? MaxML : matchLength]+1); + + if (litLength) { + /* literals */ + U32 u; + stats->priceLiteral += litLength * ZSTD_highbit(stats->litSum+1); + for (u=0; u < litLength; u++) + stats->priceLiteral -= ZSTD_highbit(stats->litFreq[literals[u]]+1); + + /* literal Length */ + stats->priceDumpsLength += ((litLength >= MaxLL)<<3) + ((litLength >= 255+MaxLL)<<4) + ((litLength>=(1<<15))<<3); + stats->priceLitLength += ZSTD_highbit(stats->litLengthSum+1) - ZSTD_highbit(stats->litLengthFreq[(litLength >= MaxLL) ? MaxLL : litLength]+1); + } else { + stats->priceLitLength += ZSTD_highbit(stats->litLengthSum+1) - ZSTD_highbit(stats->litLengthFreq[0]+1); + } + + + if (offset == 0) stats->totalRepSum++; + stats->totalSeqSum++; + stats->totalMatchSum += matchLength; + stats->totalLitSum += litLength; + + U32 u; + /* literals */ + stats->litSum += litLength; + for (u=0; u < litLength; u++) + stats->litFreq[literals[u]]++; + + /* literal Length */ + stats->litLengthSum++; + if (litLength >= MaxLL) + stats->litLengthFreq[MaxLL]++; + else + stats->litLengthFreq[litLength]++; + + /* match offset */ + stats->offCodeSum++; + stats->offCodeFreq[offCode]++; + + /* match Length */ + stats->matchLengthSum++; + if (matchLength >= MaxML) + stats->matchLengthFreq[MaxML]++; + else + stats->matchLengthFreq[matchLength]++; +} + + + +#if defined (__cplusplus) +} +#endif + +#endif /* ZSTD_STATIC_H */ From 5553442376fd3c3d86ffa1cb7ba2899b7b013729 Mon Sep 17 00:00:00 2001 From: inikep Date: Fri, 25 Mar 2016 13:35:33 +0100 Subject: [PATCH 11/20] fixed compilation with ZSTD_NOBENCH --- programs/zstdcli.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/programs/zstdcli.c b/programs/zstdcli.c index 0c060bdbe..022d75131 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -333,14 +333,15 @@ int main(int argCount, const char** argv) /* Pause at the end (-p) or set an additional param (-p#) (hidden option) */ case 'p': argument++; +#ifndef ZSTD_NOBENCH if ((*argument>='0') && (*argument<='9')) { int additionalParam = 0; while ((*argument >= '0') && (*argument <= '9')) additionalParam *= 10, additionalParam += *argument++ - '0'; BMK_setAdditionalParam(additionalParam); - } else { + } else +#endif main_pause=1; - } break; /* unknown command */ default : CLEAN_RETURN(badusage(programName)); From 97c88e716cbb1f3d63692160be53ed2f5845b8da Mon Sep 17 00:00:00 2001 From: inikep Date: Fri, 25 Mar 2016 13:39:14 +0100 Subject: [PATCH 12/20] fix for -Wshorten-64-to-32 warning in ZSTD_statsUpdatePrices --- lib/zstd_internal.h | 2 +- lib/zstd_stats.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/zstd_internal.h b/lib/zstd_internal.h index 3561291f2..c0ff68ea8 100644 --- a/lib/zstd_internal.h +++ b/lib/zstd_internal.h @@ -183,7 +183,7 @@ typedef struct { MEM_STATIC void ZSTD_statsPrint(ZSTD_stats_t* stats, U32 searchLength) { (void)stats; (void)searchLength; }; MEM_STATIC void ZSTD_statsInit(ZSTD_stats_t* stats) { (void)stats; }; MEM_STATIC void ZSTD_statsResetFreqs(ZSTD_stats_t* stats) { (void)stats; }; - MEM_STATIC void ZSTD_statsUpdatePrices(ZSTD_stats_t* stats, U32 litLength, const BYTE* literals, U32 offset, U32 matchLength) { (void)stats; (void)litLength; (void)literals; (void)offset; (void)matchLength; }; + MEM_STATIC void ZSTD_statsUpdatePrices(ZSTD_stats_t* stats, size_t litLength, const BYTE* literals, size_t offset, size_t matchLength) { (void)stats; (void)litLength; (void)literals; (void)offset; (void)matchLength; }; #endif typedef struct { diff --git a/lib/zstd_stats.h b/lib/zstd_stats.h index 8d70191c4..d0189f872 100644 --- a/lib/zstd_stats.h +++ b/lib/zstd_stats.h @@ -99,7 +99,7 @@ MEM_STATIC void ZSTD_statsResetFreqs(ZSTD_stats_t* stats) stats->offCodeFreq[u] = 1; } -MEM_STATIC void ZSTD_statsUpdatePrices(ZSTD_stats_t* stats, U32 litLength, const BYTE* literals, U32 offset, U32 matchLength) +MEM_STATIC void ZSTD_statsUpdatePrices(ZSTD_stats_t* stats, size_t litLength, const BYTE* literals, size_t offset, size_t matchLength) { /* offset */ BYTE offCode = offset ? (BYTE)ZSTD_highbit(offset+1) + 1 : 0; From 06f793a3abd63744118b7d80beaa7df99dd43498 Mon Sep 17 00:00:00 2001 From: inikep Date: Tue, 29 Mar 2016 11:17:58 +0200 Subject: [PATCH 13/20] fix for BMK_clockSpan on Windows --- programs/bench.c | 45 +++++++++++++++++++++++++++------------------ 1 file changed, 27 insertions(+), 18 deletions(-) diff --git a/programs/bench.c b/programs/bench.c index 1468bcb2b..8cdb8051e 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -162,9 +162,17 @@ void BMK_SetBlockSize(size_t blockSize) /* ******************************************************** * Private functions **********************************************************/ -static clock_t BMK_clockSpan( clock_t clockStart ) +typedef clock_t BMK_time_t; + +static BMK_time_t BMK_getTime() { - return clock() - clockStart; /* works even if overflow, span limited to <= ~30mn */ + return clock(); +} + +/* returns time span in nanoseconds */ +static U64 BMK_clockSpan( BMK_time_t clockStart ) +{ + return 1000000ULL * (BMK_getTime() - clockStart) / CLOCKS_PER_SEC; } @@ -265,20 +273,21 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize, double fastestC = 100000000., fastestD = 100000000.; double ratio = 0.; U64 crcCheck = 0; - clock_t coolTime = clock(); + BMK_time_t coolTime = BMK_getTime(); U32 testNb; DISPLAYLEVEL(2, "\r%79s\r", ""); for (testNb = 1; testNb <= (g_nbIterations + !g_nbIterations); testNb++) { int nbLoops; - clock_t clockStart, clockSpan; - clock_t const clockLoop = g_nbIterations ? TIMELOOP_S * CLOCKS_PER_SEC : 10; + BMK_time_t clockStart; + U64 clockSpan; + U64 const clockLoop = g_nbIterations ? TIMELOOP_S*1000000ULL : 10; /* overheat protection */ - if (BMK_clockSpan(coolTime) > ACTIVEPERIOD_S * CLOCKS_PER_SEC) { + if (BMK_clockSpan(coolTime) > ACTIVEPERIOD_S*1000000ULL) { DISPLAY("\rcooling down ... \r"); BMK_sleep(COOLPERIOD_S); - coolTime = clock(); + coolTime = BMK_getTime(); } /* Compression */ @@ -286,9 +295,9 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize, memset(compressedBuffer, 0xE5, maxCompressedSize); /* warm up and erase result buffer */ mili_sleep(1); /* give processor time to other processes */ - clockStart = clock(); - while (clock() == clockStart); - clockStart = clock(); + clockStart = BMK_getTime(); + while (BMK_getTime() == clockStart); + clockStart = BMK_getTime(); for (nbLoops = 0 ; BMK_clockSpan(clockStart) < clockLoop ; nbLoops++) { U32 blockNb; @@ -308,16 +317,16 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize, ratio = (double)srcSize / (double)cSize; DISPLAYLEVEL(2, "%2i-%-17.17s :%10u ->%10u (%5.3f),%6.1f MB/s\r", testNb, displayName, (U32)srcSize, (U32)cSize, ratio, - (double)srcSize / 1000000. / (fastestC / CLOCKS_PER_SEC) ); + (double)srcSize / fastestC ); #if 1 /* Decompression */ memset(resultBuffer, 0xD6, srcSize); /* warm result buffer */ mili_sleep(1); /* give processor time to other processes */ - clockStart = clock(); - while (clock() == clockStart); - clockStart = clock(); + clockStart = BMK_getTime(); + while (BMK_getTime() == clockStart); + clockStart = BMK_getTime(); for (nbLoops = 0 ; BMK_clockSpan(clockStart) < clockLoop ; nbLoops++) { U32 blockNb; @@ -338,8 +347,8 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize, if ((double)clockSpan < fastestD*nbLoops) fastestD = (double)clockSpan / nbLoops; DISPLAYLEVEL(2, "%2i-%-17.17s :%10u ->%10u (%5.3f),%6.1f MB/s ,%6.1f MB/s\r", testNb, displayName, (U32)srcSize, (U32)cSize, ratio, - (double)srcSize / 1000000. / (fastestC / CLOCKS_PER_SEC), - (double)srcSize / 1000000. / (fastestD / CLOCKS_PER_SEC) ); + (double)srcSize / fastestC, + (double)srcSize / fastestD ); /* CRC Checking */ _findError: @@ -372,8 +381,8 @@ _findError: if (crcOrig == crcCheck) { result->ratio = ratio; result->cSize = cSize; - result->cSpeed = (double)srcSize / 1000000. / (fastestC / CLOCKS_PER_SEC); - result->dSpeed = (double)srcSize / 1000000. / (fastestD / CLOCKS_PER_SEC); + result->cSpeed = (double)srcSize / fastestC; + result->dSpeed = (double)srcSize / fastestD; } DISPLAYLEVEL(2, "%2i#\n", cLevel); } /* Bench */ From 4c12f232ec0c55d9b77d7f49826a8eaa47d14262 Mon Sep 17 00:00:00 2001 From: inikep Date: Tue, 29 Mar 2016 14:52:13 +0200 Subject: [PATCH 14/20] support for QueryPerformanceCounter and clock_gettime --- programs/bench.c | 107 +++++++++++++++++++++++++++++++++++------------ 1 file changed, 81 insertions(+), 26 deletions(-) diff --git a/programs/bench.c b/programs/bench.c index 8cdb8051e..0de582380 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -57,7 +57,7 @@ # include # include /* setpriority */ # define BMK_sleep(s) sleep(s) -# define mili_sleep(mili) { struct timespec t; t.tv_sec=0; t.tv_nsec=mili*1000000L; nanosleep(&t, NULL); } +# define mili_sleep(mili) { struct timespec t; t.tv_sec=0; t.tv_nsec=mili*1000000ULL; nanosleep(&t, NULL); } # define setHighPriority() setpriority(PRIO_PROCESS, 0, -20) #elif defined(_WIN32) # include @@ -70,6 +70,49 @@ # define setHighPriority() /* disabled */ #endif +/* +Windows QueryPerformanceCounter resolution = 410 nanosec +Windows clock() resolution = 1000000 nanosec +VirtualBox Ubuntu clock() resolution = 1000 nanosec +VirtualBox Ubuntu clock_gettime() resolution = 100-280 nanosec +fizzle clock() resolution = 1000 nanosec +fizzle clock_gettime() resolution = 100-280 nanosec +*/ + +#if !defined(_WIN32) && (defined(__unix__) || defined(__unix) || (defined(__APPLE__) && defined(__MACH__))) + +#if 0 + typedef clock_t BMK_time_t; +# define BMK_TIME_FUNCTION "clock()" +# define BMK_initTimer(ticksPerSecond) (void)ticksPerSecond +# define BMK_getTime(x) x = clock() +# define BMK_getSpanTimeMicro(ticksPerSecond, clockStart, clockEnd) (1000000ULL * (clockEnd - clockStart) / CLOCKS_PER_SEC) +# define BMK_getSpanTimeNano(ticksPerSecond, clockStart, clockEnd) (1000000000ULL * (clockEnd - clockStart) / CLOCKS_PER_SEC) +#else + typedef struct timespec BMK_time_t; +# define BMK_TIME_FUNCTION "clock_gettime" +# define BMK_initTimer(ticksPerSecond) (void)ticksPerSecond +# define BMK_getTime(x) if (clock_gettime(CLOCK_MONOTONIC, &x) == -1 ){ fprintf(stderr, "ERROR: clock_gettime error\n"); } +# define BMK_getSpanTimeMicro(ticksPerSecond, clockStart, clockEnd) (1000000ULL*( clockEnd.tv_sec - clockStart.tv_sec ) + ( clockEnd.tv_nsec - clockStart.tv_nsec ) / 1000) +# define BMK_getSpanTimeNano(ticksPerSecond, clockStart, clockEnd) (1000000000ULL*( clockEnd.tv_sec - clockStart.tv_sec ) + ( clockEnd.tv_nsec - clockStart.tv_nsec )) +#endif + +#elif defined(_WIN32) + typedef LARGE_INTEGER BMK_time_t; +# define BMK_TIME_FUNCTION "QueryPerformanceFrequency" +# define BMK_initTimer(x) if (!QueryPerformanceFrequency(&x)) { fprintf(stderr, "ERROR: QueryPerformance not present\n"); } +# define BMK_getTime(x) QueryPerformanceCounter(&x) +# define BMK_getSpanTimeMicro(ticksPerSecond, clockStart, clockEnd) (1000000ULL*(clockEnd.QuadPart - clockStart.QuadPart)/ticksPerSecond.QuadPart) +# define BMK_getSpanTimeNano(ticksPerSecond, clockStart, clockEnd) (1000000000ULL*(clockEnd.QuadPart - clockStart.QuadPart)/ticksPerSecond.QuadPart) +#else + typedef int BMK_time_t; +# define BMK_TIME_FUNCTION "None" +# define BMK_initTimer(ticksPerSecond) (void)ticksPerSecond +# define BMK_getTimeMicro(clockStart) clockStart=1 +# define BMK_getSpanTimeMicro(ticksPerSecond, clockStart, clockEnd) (TIMELOOP_S*1000000ULL+clockEnd-clockStart) +# define BMK_getSpanTimeNano(ticksPerSecond, clockStart, clockEnd) (TIMELOOP_S*1000000000ULL+clockEnd-clockStart) +#endif + #include "mem.h" #include "zstd_static.h" #include "zstd_internal.h" /* ZSTD_compressBegin_targetSrcSize */ @@ -162,20 +205,16 @@ void BMK_SetBlockSize(size_t blockSize) /* ******************************************************** * Private functions **********************************************************/ -typedef clock_t BMK_time_t; - -static BMK_time_t BMK_getTime() +/* returns time span in microseconds */ +static U64 BMK_clockSpan( BMK_time_t clockStart, BMK_time_t ticksPerSecond ) { - return clock(); + BMK_time_t clockEnd; + + (void)ticksPerSecond; + BMK_getTime(clockEnd); + return BMK_getSpanTimeMicro(ticksPerSecond, clockStart, clockEnd); } -/* returns time span in nanoseconds */ -static U64 BMK_clockSpan( BMK_time_t clockStart ) -{ - return 1000000ULL * (BMK_getTime() - clockStart) / CLOCKS_PER_SEC; -} - - static U64 BMK_getFileSize(const char* infilename) { int r; @@ -235,6 +274,7 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize, U64 const crcOrig = XXH64(srcBuffer, srcSize, 0); U32 nbBlocks; + BMK_time_t ticksPerSecond; /* checks */ if (!compressedBuffer || !resultBuffer || !blockTable || !refCtx || !ctx || !refDCtx || !dctx) @@ -242,6 +282,18 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize, /* init */ if (strlen(displayName)>17) displayName += strlen(displayName)-17; /* can only display 17 characters */ + BMK_initTimer(ticksPerSecond); + + { + BMK_time_t clockStart, clockEnd; + + BMK_getTime(clockStart); + do { BMK_getTime(clockEnd); } + while (BMK_getSpanTimeNano(ticksPerSecond, clockStart, clockEnd) == 0); + + printf(BMK_TIME_FUNCTION " resolution = %d nanosec\n", (int)BMK_getSpanTimeNano(ticksPerSecond, clockStart, clockEnd)); + } + /* Init blockTable data */ { const char* srcPtr = (const char*)srcBuffer; @@ -273,21 +325,22 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize, double fastestC = 100000000., fastestD = 100000000.; double ratio = 0.; U64 crcCheck = 0; - BMK_time_t coolTime = BMK_getTime(); + BMK_time_t coolTime; U32 testNb; + BMK_getTime(coolTime); DISPLAYLEVEL(2, "\r%79s\r", ""); for (testNb = 1; testNb <= (g_nbIterations + !g_nbIterations); testNb++) { int nbLoops; - BMK_time_t clockStart; + BMK_time_t clockStart, clockEnd; U64 clockSpan; U64 const clockLoop = g_nbIterations ? TIMELOOP_S*1000000ULL : 10; /* overheat protection */ - if (BMK_clockSpan(coolTime) > ACTIVEPERIOD_S*1000000ULL) { + if (BMK_clockSpan(coolTime, ticksPerSecond) > ACTIVEPERIOD_S*1000000ULL) { DISPLAY("\rcooling down ... \r"); BMK_sleep(COOLPERIOD_S); - coolTime = BMK_getTime(); + BMK_getTime(coolTime); } /* Compression */ @@ -295,11 +348,12 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize, memset(compressedBuffer, 0xE5, maxCompressedSize); /* warm up and erase result buffer */ mili_sleep(1); /* give processor time to other processes */ - clockStart = BMK_getTime(); - while (BMK_getTime() == clockStart); - clockStart = BMK_getTime(); + BMK_getTime(clockStart); + do { BMK_getTime(clockEnd); } + while (BMK_getSpanTimeNano(ticksPerSecond, clockStart, clockEnd) == 0); + BMK_getTime(clockStart); - for (nbLoops = 0 ; BMK_clockSpan(clockStart) < clockLoop ; nbLoops++) { + for (nbLoops = 0 ; BMK_clockSpan(clockStart, ticksPerSecond) < clockLoop ; nbLoops++) { U32 blockNb; ZSTD_compressBegin_targetSrcSize(refCtx, dictBuffer, dictBufferSize, blockSize, cLevel); for (blockNb=0; blockNb%10u (%5.3f),%6.1f MB/s ,%6.1f MB/s\r", testNb, displayName, (U32)srcSize, (U32)cSize, ratio, From 4611d11fea2402e45edd7093a73b6e7f15c6f986 Mon Sep 17 00:00:00 2001 From: inikep Date: Tue, 29 Mar 2016 15:52:38 +0200 Subject: [PATCH 15/20] added -lrt required by clock_gettime --- programs/Makefile | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/programs/Makefile b/programs/Makefile index 6aca9768e..b81bfeba2 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -48,6 +48,7 @@ CPPFLAGS= -I../lib -DZSTD_VERSION=\"$(VERSION)\" CFLAGS ?= -O3 # -falign-loops=32 # not always beneficial CFLAGS += -std=c99 -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wstrict-aliasing=1 -Wswitch-enum -Wstrict-prototypes -Wundef FLAGS = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) $(MOREFLAGS) +LDFLAGS ?= -lrt BINDIR = $(PREFIX)/bin MANDIR = $(PREFIX)/share/man/man1 @@ -86,11 +87,11 @@ all: zstd zstd32 fullbench fullbench32 fuzzer fuzzer32 zbufftest zbufftest32 par zstd : $(ZSTD_FILES) $(ZSTD_FILES_LEGACY) $(ZSTDDIR)/zbuff.c $(ZSTDDIR)/zdict.c $(ZSTDDIR)/divsufsort.c \ zstdcli.c fileio.c bench.c xxhash.c datagen.c dibio.c - $(CC) $(FLAGS) -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT) $^ -o $@$(EXT) + $(CC) $(FLAGS) -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT) $^ $(LDFLAGS) -o $@$(EXT) zstd32: $(ZSTD_FILES) $(ZSTD_FILES_LEGACY) $(ZSTDDIR)/zbuff.c $(ZSTDDIR)/zdict.c $(ZSTDDIR)/divsufsort.c \ zstdcli.c fileio.c bench.c xxhash.c datagen.c dibio.c - $(CC) -m32 $(FLAGS) -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT) $^ -o $@$(EXT) + $(CC) -m32 $(FLAGS) -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT) $^ $(LDFLAGS) -o $@$(EXT) zstd_nolegacy : $(MAKE) zstd ZSTD_LEGACY_SUPPORT=0 From 33ad2512368bac6c06d4242e9df2c921c2049e63 Mon Sep 17 00:00:00 2001 From: inikep Date: Wed, 30 Mar 2016 09:55:37 +0200 Subject: [PATCH 16/20] removed clock_gettime (not portable) --- programs/Makefile | 5 ++--- programs/bench.c | 34 ---------------------------------- 2 files changed, 2 insertions(+), 37 deletions(-) diff --git a/programs/Makefile b/programs/Makefile index b81bfeba2..6aca9768e 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -48,7 +48,6 @@ CPPFLAGS= -I../lib -DZSTD_VERSION=\"$(VERSION)\" CFLAGS ?= -O3 # -falign-loops=32 # not always beneficial CFLAGS += -std=c99 -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wstrict-aliasing=1 -Wswitch-enum -Wstrict-prototypes -Wundef FLAGS = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) $(MOREFLAGS) -LDFLAGS ?= -lrt BINDIR = $(PREFIX)/bin MANDIR = $(PREFIX)/share/man/man1 @@ -87,11 +86,11 @@ all: zstd zstd32 fullbench fullbench32 fuzzer fuzzer32 zbufftest zbufftest32 par zstd : $(ZSTD_FILES) $(ZSTD_FILES_LEGACY) $(ZSTDDIR)/zbuff.c $(ZSTDDIR)/zdict.c $(ZSTDDIR)/divsufsort.c \ zstdcli.c fileio.c bench.c xxhash.c datagen.c dibio.c - $(CC) $(FLAGS) -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT) $^ $(LDFLAGS) -o $@$(EXT) + $(CC) $(FLAGS) -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT) $^ -o $@$(EXT) zstd32: $(ZSTD_FILES) $(ZSTD_FILES_LEGACY) $(ZSTDDIR)/zbuff.c $(ZSTDDIR)/zdict.c $(ZSTDDIR)/divsufsort.c \ zstdcli.c fileio.c bench.c xxhash.c datagen.c dibio.c - $(CC) -m32 $(FLAGS) -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT) $^ $(LDFLAGS) -o $@$(EXT) + $(CC) -m32 $(FLAGS) -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT) $^ -o $@$(EXT) zstd_nolegacy : $(MAKE) zstd ZSTD_LEGACY_SUPPORT=0 diff --git a/programs/bench.c b/programs/bench.c index 0de582380..28027f4ad 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -70,43 +70,20 @@ # define setHighPriority() /* disabled */ #endif -/* -Windows QueryPerformanceCounter resolution = 410 nanosec -Windows clock() resolution = 1000000 nanosec -VirtualBox Ubuntu clock() resolution = 1000 nanosec -VirtualBox Ubuntu clock_gettime() resolution = 100-280 nanosec -fizzle clock() resolution = 1000 nanosec -fizzle clock_gettime() resolution = 100-280 nanosec -*/ - #if !defined(_WIN32) && (defined(__unix__) || defined(__unix) || (defined(__APPLE__) && defined(__MACH__))) - -#if 0 typedef clock_t BMK_time_t; -# define BMK_TIME_FUNCTION "clock()" # define BMK_initTimer(ticksPerSecond) (void)ticksPerSecond # define BMK_getTime(x) x = clock() # define BMK_getSpanTimeMicro(ticksPerSecond, clockStart, clockEnd) (1000000ULL * (clockEnd - clockStart) / CLOCKS_PER_SEC) # define BMK_getSpanTimeNano(ticksPerSecond, clockStart, clockEnd) (1000000000ULL * (clockEnd - clockStart) / CLOCKS_PER_SEC) -#else - typedef struct timespec BMK_time_t; -# define BMK_TIME_FUNCTION "clock_gettime" -# define BMK_initTimer(ticksPerSecond) (void)ticksPerSecond -# define BMK_getTime(x) if (clock_gettime(CLOCK_MONOTONIC, &x) == -1 ){ fprintf(stderr, "ERROR: clock_gettime error\n"); } -# define BMK_getSpanTimeMicro(ticksPerSecond, clockStart, clockEnd) (1000000ULL*( clockEnd.tv_sec - clockStart.tv_sec ) + ( clockEnd.tv_nsec - clockStart.tv_nsec ) / 1000) -# define BMK_getSpanTimeNano(ticksPerSecond, clockStart, clockEnd) (1000000000ULL*( clockEnd.tv_sec - clockStart.tv_sec ) + ( clockEnd.tv_nsec - clockStart.tv_nsec )) -#endif - #elif defined(_WIN32) typedef LARGE_INTEGER BMK_time_t; -# define BMK_TIME_FUNCTION "QueryPerformanceFrequency" # define BMK_initTimer(x) if (!QueryPerformanceFrequency(&x)) { fprintf(stderr, "ERROR: QueryPerformance not present\n"); } # define BMK_getTime(x) QueryPerformanceCounter(&x) # define BMK_getSpanTimeMicro(ticksPerSecond, clockStart, clockEnd) (1000000ULL*(clockEnd.QuadPart - clockStart.QuadPart)/ticksPerSecond.QuadPart) # define BMK_getSpanTimeNano(ticksPerSecond, clockStart, clockEnd) (1000000000ULL*(clockEnd.QuadPart - clockStart.QuadPart)/ticksPerSecond.QuadPart) #else typedef int BMK_time_t; -# define BMK_TIME_FUNCTION "None" # define BMK_initTimer(ticksPerSecond) (void)ticksPerSecond # define BMK_getTimeMicro(clockStart) clockStart=1 # define BMK_getSpanTimeMicro(ticksPerSecond, clockStart, clockEnd) (TIMELOOP_S*1000000ULL+clockEnd-clockStart) @@ -284,17 +261,6 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize, if (strlen(displayName)>17) displayName += strlen(displayName)-17; /* can only display 17 characters */ BMK_initTimer(ticksPerSecond); - { - BMK_time_t clockStart, clockEnd; - - BMK_getTime(clockStart); - do { BMK_getTime(clockEnd); } - while (BMK_getSpanTimeNano(ticksPerSecond, clockStart, clockEnd) == 0); - - printf(BMK_TIME_FUNCTION " resolution = %d nanosec\n", (int)BMK_getSpanTimeNano(ticksPerSecond, clockStart, clockEnd)); - } - - /* Init blockTable data */ { const char* srcPtr = (const char*)srcBuffer; char* cPtr = (char*)compressedBuffer; From 1c556a3838ec6f32d1574f49787eb1fb46ac8db7 Mon Sep 17 00:00:00 2001 From: inikep Date: Wed, 30 Mar 2016 10:59:48 +0200 Subject: [PATCH 17/20] ticksPerSecond=0 --- programs/bench.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/programs/bench.c b/programs/bench.c index 28027f4ad..dec575893 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -72,7 +72,7 @@ #if !defined(_WIN32) && (defined(__unix__) || defined(__unix) || (defined(__APPLE__) && defined(__MACH__))) typedef clock_t BMK_time_t; -# define BMK_initTimer(ticksPerSecond) (void)ticksPerSecond +# define BMK_initTimer(ticksPerSecond) ticksPerSecond=0 # define BMK_getTime(x) x = clock() # define BMK_getSpanTimeMicro(ticksPerSecond, clockStart, clockEnd) (1000000ULL * (clockEnd - clockStart) / CLOCKS_PER_SEC) # define BMK_getSpanTimeNano(ticksPerSecond, clockStart, clockEnd) (1000000000ULL * (clockEnd - clockStart) / CLOCKS_PER_SEC) @@ -84,7 +84,7 @@ # define BMK_getSpanTimeNano(ticksPerSecond, clockStart, clockEnd) (1000000000ULL*(clockEnd.QuadPart - clockStart.QuadPart)/ticksPerSecond.QuadPart) #else typedef int BMK_time_t; -# define BMK_initTimer(ticksPerSecond) (void)ticksPerSecond +# define BMK_initTimer(ticksPerSecond) ticksPerSecond=0 # define BMK_getTimeMicro(clockStart) clockStart=1 # define BMK_getSpanTimeMicro(ticksPerSecond, clockStart, clockEnd) (TIMELOOP_S*1000000ULL+clockEnd-clockStart) # define BMK_getSpanTimeNano(ticksPerSecond, clockStart, clockEnd) (TIMELOOP_S*1000000000ULL+clockEnd-clockStart) From 227dd4bc7426be73a9656be6e55e1434d16e9018 Mon Sep 17 00:00:00 2001 From: inikep Date: Mon, 4 Apr 2016 14:34:02 +0200 Subject: [PATCH 18/20] changed price estimation for optimal parser --- lib/zstd_opt.h | 84 ++++++++++++++++++++++++++++++++------------------ 1 file changed, 54 insertions(+), 30 deletions(-) diff --git a/lib/zstd_opt.h b/lib/zstd_opt.h index 67a26662c..bbb4393f1 100644 --- a/lib/zstd_opt.h +++ b/lib/zstd_opt.h @@ -111,9 +111,18 @@ FORCE_INLINE U32 ZSTD_getLiteralPrice(seqStore_t* seqStorePtr, U32 litLength, co price -= ZSTD_highbit(seqStorePtr->litFreq[literals[u]]+1); /* literal Length */ - price += ((litLength >= MaxLL)<<3) + ((litLength >= 255+MaxLL)<<4) + ((litLength>=(1<<15))<<3); - if (litLength >= MaxLL) litLength = MaxLL; - price += seqStorePtr->log2litLengthSum - ZSTD_highbit(seqStorePtr->litLengthFreq[litLength]+1); + { static const BYTE LL_Code[64] = { 0, 1, 2, 3, 4, 5, 6, 7, + 8, 9, 10, 11, 12, 13, 14, 15, + 16, 16, 17, 17, 18, 18, 19, 19, + 20, 20, 20, 20, 21, 21, 21, 21, + 22, 22, 22, 22, 22, 22, 22, 22, + 23, 23, 23, 23, 23, 23, 23, 23, + 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24 }; + const BYTE LL_deltaCode = 19; + const BYTE llCode = (litLength>63) ? (BYTE)ZSTD_highbit(litLength) + LL_deltaCode : LL_Code[litLength]; + price += LL_bits[llCode] + seqStorePtr->log2litLengthSum - ZSTD_highbit(seqStorePtr->litLengthFreq[llCode]+1); + } return price; } @@ -122,25 +131,24 @@ FORCE_INLINE U32 ZSTD_getLiteralPrice(seqStore_t* seqStorePtr, U32 litLength, co FORCE_INLINE U32 ZSTD_getPrice(seqStore_t* seqStorePtr, U32 litLength, const BYTE* literals, U32 offset, U32 matchLength) { /* offset */ - BYTE offCode = offset ? (BYTE)ZSTD_highbit(offset+1) + 1 : 0; - U32 price = (offCode-1) + (!offCode) + seqStorePtr->log2offCodeSum - ZSTD_highbit(seqStorePtr->offCodeFreq[offCode]+1); + BYTE offCode = (BYTE)ZSTD_highbit(offset+1); + U32 price = offCode + seqStorePtr->log2offCodeSum - ZSTD_highbit(seqStorePtr->offCodeFreq[offCode]+1); /* match Length */ - price += ((matchLength >= MaxML)<<3) + ((matchLength >= 255+MaxML)<<4) + ((matchLength>=(1<<15))<<3); - if (matchLength >= MaxML) matchLength = MaxML; - price += ZSTD_getLiteralPrice(seqStorePtr, litLength, literals) + seqStorePtr->log2matchLengthSum - ZSTD_highbit(seqStorePtr->matchLengthFreq[matchLength]+1); - -#if ZSTD_OPT_DEBUG == 333 - switch (seqStorePtr->) { - default: - case 0: - return 1 + price + ((seqStorePtr->litSum>>5) / seqStorePtr->litLengthSum) + ((seqStorePtr->litSum<<1) / (seqStorePtr->litSum + seqStorePtr->matchSum)); - case 1: - return 1 + price; + { static const BYTE ML_Code[128] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, + 32, 32, 33, 33, 34, 34, 35, 35, 36, 36, 36, 36, 37, 37, 37, 37, + 38, 38, 38, 38, 38, 38, 38, 38, 39, 39, 39, 39, 39, 39, 39, 39, + 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, + 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, + 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, + 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42 }; + const BYTE ML_deltaCode = 36; + const BYTE mlCode = (matchLength>127) ? (BYTE)ZSTD_highbit(matchLength) + ML_deltaCode : ML_Code[matchLength]; + price += ML_bits[mlCode] + seqStorePtr->log2matchLengthSum - ZSTD_highbit(seqStorePtr->matchLengthFreq[mlCode]+1); } -#else - return price + seqStorePtr->factor; -#endif + + return price + ZSTD_getLiteralPrice(seqStorePtr, litLength, literals) + seqStorePtr->factor; } @@ -154,23 +162,39 @@ MEM_STATIC void ZSTD_updatePrice(seqStore_t* seqStorePtr, U32 litLength, const B seqStorePtr->litFreq[literals[u]]++; /* literal Length */ - seqStorePtr->litLengthSum++; - if (litLength >= MaxLL) - seqStorePtr->litLengthFreq[MaxLL]++; - else - seqStorePtr->litLengthFreq[litLength]++; + { static const BYTE LL_Code[64] = { 0, 1, 2, 3, 4, 5, 6, 7, + 8, 9, 10, 11, 12, 13, 14, 15, + 16, 16, 17, 17, 18, 18, 19, 19, + 20, 20, 20, 20, 21, 21, 21, 21, + 22, 22, 22, 22, 22, 22, 22, 22, + 23, 23, 23, 23, 23, 23, 23, 23, + 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24 }; + const BYTE LL_deltaCode = 19; + const BYTE llCode = (litLength>63) ? (BYTE)ZSTD_highbit(litLength) + LL_deltaCode : LL_Code[litLength]; + seqStorePtr->litLengthFreq[llCode]++; + seqStorePtr->litLengthSum++; + } /* match offset */ seqStorePtr->offCodeSum++; - BYTE offCode = offset ? (BYTE)ZSTD_highbit(offset+1) + 1 : 0; + BYTE offCode = ZSTD_highbit(offset+1); seqStorePtr->offCodeFreq[offCode]++; /* match Length */ - seqStorePtr->matchLengthSum++; - if (matchLength >= MaxML) - seqStorePtr->matchLengthFreq[MaxML]++; - else - seqStorePtr->matchLengthFreq[matchLength]++; + { static const BYTE ML_Code[128] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, + 32, 32, 33, 33, 34, 34, 35, 35, 36, 36, 36, 36, 37, 37, 37, 37, + 38, 38, 38, 38, 38, 38, 38, 38, 39, 39, 39, 39, 39, 39, 39, 39, + 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, + 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, + 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, + 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42 }; + const BYTE ML_deltaCode = 36; + const BYTE mlCode = (matchLength>127) ? (BYTE)ZSTD_highbit(matchLength) + ML_deltaCode : ML_Code[matchLength]; + seqStorePtr->matchLengthFreq[mlCode]++; + seqStorePtr->matchLengthSum++; + } ZSTD_setLog2Prices(seqStorePtr); } From cb70c8ab90f3cd439ff37dcd4675eee892b2198c Mon Sep 17 00:00:00 2001 From: inikep Date: Mon, 4 Apr 2016 15:43:45 +0200 Subject: [PATCH 19/20] fixed -Werror,-Wconversion warning --- lib/zstd_opt.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/zstd_opt.h b/lib/zstd_opt.h index d868d3cd2..145570031 100644 --- a/lib/zstd_opt.h +++ b/lib/zstd_opt.h @@ -178,7 +178,7 @@ MEM_STATIC void ZSTD_updatePrice(seqStore_t* seqStorePtr, U32 litLength, const B /* match offset */ seqStorePtr->offCodeSum++; - BYTE offCode = ZSTD_highbit(offset+1); + BYTE offCode = (BYTE)ZSTD_highbit(offset+1); seqStorePtr->offCodeFreq[offCode]++; /* match Length */ From 19140cd80ebd50c47d25363b63d6753b07d8e5e0 Mon Sep 17 00:00:00 2001 From: inikep Date: Tue, 5 Apr 2016 08:52:57 +0200 Subject: [PATCH 20/20] zstd_stats.h moved to .debug/ --- lib/{ => .debug}/zstd_stats.h | 0 lib/zstd_internal.h | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename lib/{ => .debug}/zstd_stats.h (100%) diff --git a/lib/zstd_stats.h b/lib/.debug/zstd_stats.h similarity index 100% rename from lib/zstd_stats.h rename to lib/.debug/zstd_stats.h diff --git a/lib/zstd_internal.h b/lib/zstd_internal.h index 49edb3109..ce36aad3b 100644 --- a/lib/zstd_internal.h +++ b/lib/zstd_internal.h @@ -193,7 +193,7 @@ typedef struct { } ZSTD_optimal_t; #if ZSTD_OPT_DEBUG == 3 - #include "zstd_stats.h" + #include ".debug/zstd_stats.h" #else typedef struct { U32 unused; } ZSTD_stats_t; MEM_STATIC void ZSTD_statsPrint(ZSTD_stats_t* stats, U32 searchLength) { (void)stats; (void)searchLength; };