ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'upstream/dev' into btopt

Browse Source 
* upstream/dev: (305 commits)
  added test for ZSTD_estimateCStreamSize()
  changed variable name, for clarity
  fixed ZSTD_estimateCStreamSize()
  shortened ZSTD_createCStream_Advanced()
  fixed symbols test
  added ZSTD_estimateDStreamSize()
  changed name frameParams into frameHeader
  regroup memory usage function declarations
  separated ZSTD_estimateCStreamSize() from ZSTD_estimateCCtxSize()
  bumped version number
  added ZSTD_estimateCDictSize() and ZSTD_estimateDDictSize()
  Updated ZSTD_freeCCtx()
  updated ZSTD_estimateCCtxSize()
  Updated ZSTD_sizeof_CCtx()
  merged CCtx and CStream as a single same object
  cli : -d and -t do not stop after a failed decompression
  added dev branch CircleCI badge
  added dev branch Appveyor badge
  keep dev branch status only
  creates a binary archive without the `programs` directory
  ...
This commit is contained in:
Nick Terrell
2017-05-10 16:49:58 -07:00
parent f35ef5c8e9 ba41b26405
commit 5f2c7213c7
149 changed files with 22690 additions and 2821 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/paramgrill.c
+52 -30
View File
@@ -58,6 +58,11 @@ static const int g_maxNbVariations = 64;
**************************************/
#define DISPLAY(...) fprintf(stderr, __VA_ARGS__)
#undef MIN
#undef MAX
#define MIN(a,b) ( (a) < (b) ? (a) : (b) )
#define MAX(a,b) ( (a) > (b) ? (a) : (b) )
/*-************************************
* Benchmark Parameters
@@ -106,7 +111,11 @@ static size_t BMK_findMaxMem(U64 requiredMem)
}
# define FUZ_rotl32(x,r) ((x << r) | (x >> (32 - r)))
static U32 FUZ_rotl32(U32 x, U32 r)
{
return ((x << r) | (x >> (32 - r)));
}
U32 FUZ_rand(U32* src)
{
const U32 prime1 = 2654435761U;
@@ -125,7 +134,7 @@ U32 FUZ_rand(U32* src)
*********************************************************/
typedef struct {
size_t cSize;
double cSpeed;
double cSpeed; /* bytes / sec */
double dSpeed;
} BMK_result_t;
@@ -141,8 +150,6 @@ typedef struct
} blockParam_t;
#define MIN(a,b) ( (a) < (b) ? (a) : (b) )
static size_t BMK_benchParam(BMK_result_t* resultPtr,
const void* srcBuffer, size_t srcSize,
ZSTD_CCtx* ctx,
@@ -165,6 +172,11 @@ static size_t BMK_benchParam(BMK_result_t* resultPtr,
char name[30] = { 0 };
U64 crcOrig;
/* init result for early exit */
resultPtr->cSize = srcSize;
resultPtr->cSpeed = 0.;
resultPtr->dSpeed = 0.;
/* Memory allocation & restrictions */
snprintf(name, 30, "Sw%02uc%02uh%02us%02ul%1ut%03uS%1u", Wlog, Clog, Hlog, Slog, Slength, Tlength, strat);
if (!compressedBuffer || !resultBuffer || !blockTable) {
@@ -206,7 +218,6 @@ static size_t BMK_benchParam(BMK_result_t* resultPtr,
size_t cSize = 0;
double fastestC = 100000000., fastestD = 100000000.;
double ratio = 0.;
U64 crcCheck = 0;
clock_t const benchStart = clock();
DISPLAY("\r%79s\r", "");
@@ -242,8 +253,8 @@ static size_t BMK_benchParam(BMK_result_t* resultPtr,
cSize = 0;
for (blockNb=0; blockNb<nbBlocks; blockNb++)
cSize += blockTable[blockNb].cSize;
if ((double)roundClock < fastestC * CLOCKS_PER_SEC * nbLoops) fastestC = ((double)roundClock / CLOCKS_PER_SEC) / nbLoops;
ratio = (double)srcSize / (double)cSize;
if ((double)roundClock < fastestC * CLOCKS_PER_SEC * nbLoops) fastestC = ((double)roundClock / CLOCKS_PER_SEC) / nbLoops;
DISPLAY("\r");
DISPLAY("%1u-%s : %9u ->", loopNb, name, (U32)srcSize);
DISPLAY(" %9u (%4.3f),%7.1f MB/s", (U32)cSize, ratio, (double)srcSize / fastestC / 1000000.);
@@ -273,18 +284,18 @@ static size_t BMK_benchParam(BMK_result_t* resultPtr,
resultPtr->dSpeed = (double)srcSize / fastestD;
/* CRC Checking */
crcCheck = XXH64(resultBuffer, srcSize, 0);
if (crcOrig!=crcCheck) {
unsigned u;
unsigned eBlockSize = (unsigned)(MIN(65536*2, blockSize));
DISPLAY("\n!!! WARNING !!! Invalid Checksum : %x != %x\n", (unsigned)crcOrig, (unsigned)crcCheck);
for (u=0; u<srcSize; u++) {
if (((const BYTE*)srcBuffer)[u] != ((BYTE*)resultBuffer)[u]) {
printf("Decoding error at pos %u (block %u, pos %u) \n", u, u / eBlockSize, u % eBlockSize);
break;
} }
break;
}
{ U64 const crcCheck = XXH64(resultBuffer, srcSize, 0);
if (crcOrig!=crcCheck) {
unsigned u;
unsigned eBlockSize = (unsigned)(MIN(65536*2, blockSize));
DISPLAY("\n!!! WARNING !!! Invalid Checksum : %x != %x\n", (unsigned)crcOrig, (unsigned)crcCheck);
for (u=0; u<srcSize; u++) {
if (((const BYTE*)srcBuffer)[u] != ((BYTE*)resultBuffer)[u]) {
printf("Decoding error at pos %u (block %u, pos %u) \n", u, u / eBlockSize, u % eBlockSize);
break;
} }
break;
} }
#endif
} }
@@ -505,8 +516,6 @@ static BYTE g_alreadyTested[PARAMTABLESIZE] = {0}; /* init to zero */
g_alreadyTested[(XXH64(sanitizeParams(p), sizeof(p), 0) >> 3) & PARAMTABLEMASK]
#define MAX(a,b) ( (a) > (b) ? (a) : (b) )
static void playAround(FILE* f, winnerInfo_t* winners,
ZSTD_compressionParameters params,
const void* srcBuffer, size_t srcSize,
@@ -711,6 +720,14 @@ int benchFiles(const char** fileNamesTable, int nbFiles)
}
static void BMK_translateAdvancedParams(ZSTD_compressionParameters params)
{
DISPLAY("--zstd=windowLog=%u,chainLog=%u,hashLog=%u,searchLog=%u,searchLength=%u,targetLength=%u,strategy=%u \n",
params.windowLog, params.chainLog, params.hashLog, params.searchLog, params.searchLength, params.targetLength, (U32)(params.strategy));
}
/* optimizeForSize():
* targetSpeed : expressed in MB/s */
int optimizeForSize(const char* inFileName, U32 targetSpeed)
{
FILE* const inFile = fopen( inFileName, "rb" );
@@ -723,8 +740,11 @@ int optimizeForSize(const char* inFileName, U32 targetSpeed)
/* Memory allocation & restrictions */
if ((U64)benchedSize > inFileSize) benchedSize = (size_t)inFileSize;
if (benchedSize < inFileSize)
DISPLAY("Not enough memory for '%s' full size; testing %i MB only...\n", inFileName, (int)(benchedSize>>20));
if (benchedSize < inFileSize) {
DISPLAY("Not enough memory for '%s' \n", inFileName);
fclose(inFile);
return 11;
}
/* Alloc */
origBuff = malloc(benchedSize);
@@ -747,10 +767,9 @@ int optimizeForSize(const char* inFileName, U32 targetSpeed)
/* bench */
DISPLAY("\r%79s\r", "");
DISPLAY("optimizing for %s - limit speed %u MB/s \n", inFileName, targetSpeed);
targetSpeed *= 1000;
targetSpeed *= 1000000;
{ ZSTD_CCtx* const ctx = ZSTD_createCCtx();
ZSTD_compressionParameters params;
winnerInfo_t winner;
BMK_result_t candidate;
const size_t blockSize = g_blockSize ? g_blockSize : benchedSize;
@@ -764,26 +783,28 @@ int optimizeForSize(const char* inFileName, U32 targetSpeed)
{ const int maxSeeds = g_noSeed ? 1 : ZSTD_maxCLevel();
int i;
for (i=1; i<=maxSeeds; i++) {
params = ZSTD_getCParams(i, blockSize, 0);
BMK_benchParam(&candidate, origBuff, benchedSize, ctx, params);
ZSTD_compressionParameters const CParams = ZSTD_getCParams(i, blockSize, 0);
BMK_benchParam(&candidate, origBuff, benchedSize, ctx, CParams);
if (candidate.cSpeed < targetSpeed)
break;
if ( (candidate.cSize < winner.result.cSize)
| ((candidate.cSize == winner.result.cSize) & (candidate.cSpeed > winner.result.cSpeed)) )
{
winner.params = params;
winner.params = CParams;
winner.result = candidate;
BMK_printWinner(stdout, i, winner.result, winner.params, benchedSize);
} }
}
BMK_printWinner(stdout, 99, winner.result, winner.params, benchedSize);
BMK_translateAdvancedParams(winner.params);
/* start tests */
{ time_t const grillStart = time(NULL);
do {
params = winner.params;
ZSTD_compressionParameters params = winner.params;
paramVariation(&params);
if ((FUZ_rand(&g_rand) & 15) == 3) params = randomParams();
if ((FUZ_rand(&g_rand) & 31) == 3) params = randomParams(); /* totally random config to improve search space */
params = ZSTD_adjustCParams(params, blockSize, 0);
/* exclude faster if already played set of params */
if (FUZ_rand(&g_rand) & ((1 << NB_TESTS_PLAYED(params))-1)) continue;
@@ -800,6 +821,7 @@ int optimizeForSize(const char* inFileName, U32 targetSpeed)
winner.params = params;
winner.result = candidate;
BMK_printWinner(stdout, 99, winner.result, winner.params, benchedSize);
BMK_translateAdvancedParams(winner.params);
}
} while (BMK_timeSpan(grillStart) < g_grillDuration_s);
}
@@ -833,7 +855,7 @@ static int usage_advanced(void)
DISPLAY( " -T# : set level 1 speed objective \n");
DISPLAY( " -B# : cut input into blocks of size # (default : single block) \n");
DISPLAY( " -i# : iteration loops [1-9](default : %i) \n", NBLOOPS);
DISPLAY( " -O# : find Optimized parameters for # target speed (default : 0) \n");
DISPLAY( " -O# : find Optimized parameters for # MB/s compression speed (default : 0) \n");
DISPLAY( " -S : Single run \n");
DISPLAY( " -P# : generated sample compressibility (default : %.1f%%) \n", COMPRESSIBILITY_DEFAULT * 100);
return 0;