From 4c4149452c8ace4afb6c87d14cd4bac491bc9de3 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 28 May 2019 16:37:03 -0700 Subject: [PATCH 1/7] fullbench can now select sample size with command `-B#` --- tests/fullbench.c | 42 +++++++++++++++++++++++++----------------- 1 file changed, 25 insertions(+), 17 deletions(-) diff --git a/tests/fullbench.c b/tests/fullbench.c index b06e2edc5..b996fc5cd 100644 --- a/tests/fullbench.c +++ b/tests/fullbench.c @@ -51,7 +51,7 @@ #define DEFAULT_CLEVEL 1 #define COMPRESSIBILITY_DEFAULT 0.50 -static const size_t g_sampleSize = 10000000; +static const size_t k_sampleSize_default = 10000000; #define TIMELOOP_NANOSEC (1*1000000000ULL) /* 1 second */ @@ -66,7 +66,6 @@ static const size_t g_sampleSize = 10000000; * Benchmark Parameters **************************************/ static unsigned g_nbIterations = NBLOOPS; -static double g_compressibility = COMPRESSIBILITY_DEFAULT; /*_******************************************************* @@ -549,21 +548,19 @@ _cleanOut: static int benchSample(U32 benchNb, + size_t benchedSize, double compressibility, int cLevel, ZSTD_compressionParameters cparams) { - size_t const benchedSize = g_sampleSize; - const char* const name = "Sample 10MiB"; - /* Allocation */ void* const origBuff = malloc(benchedSize); if (!origBuff) { DISPLAY("\nError: not enough memory!\n"); return 12; } /* Fill buffer */ - RDG_genBuffer(origBuff, benchedSize, g_compressibility, 0.0, 0); + RDG_genBuffer(origBuff, benchedSize, compressibility, 0.0, 0); /* bench */ DISPLAY("\r%70s\r", ""); - DISPLAY(" %s : \n", name); + DISPLAY(" Sample %u bytes : \n", (unsigned)benchedSize); if (benchNb) { benchMem(benchNb, origBuff, benchedSize, cLevel, cparams); } else { /* 0 == run all tests */ @@ -696,10 +693,11 @@ static int usage_advanced(const char* exename) usage(exename); DISPLAY( "\nAdvanced options :\n"); DISPLAY( " -b# : test only function # \n"); - DISPLAY( " -i# : iteration loops [1-9](default : %i)\n", NBLOOPS); - DISPLAY( " -P# : sample compressibility (default : %.1f%%)\n", COMPRESSIBILITY_DEFAULT * 100); DISPLAY( " -l# : benchmark functions at that compression level (default : %i)\n", DEFAULT_CLEVEL); DISPLAY( " --zstd : custom parameter selection. Format same as zstdcli \n"); + DISPLAY( " -P# : sample compressibility (default : %.1f%%)\n", COMPRESSIBILITY_DEFAULT * 100); + DISPLAY( " -B# : sample size (default : %u)\n", (unsigned)k_sampleSize_default); + DISPLAY( " -i# : iteration loops [1-9](default : %i)\n", NBLOOPS); return 0; } @@ -718,6 +716,8 @@ int main(int argc, const char** argv) U32 benchNb = 0, main_pause = 0; int cLevel = DEFAULT_CLEVEL; ZSTD_compressionParameters cparams = ZSTD_getCParams(cLevel, 0, 0); + size_t sampleSize = k_sampleSize_default; + double compressibility = COMPRESSIBILITY_DEFAULT; DISPLAY(WELCOME_MESSAGE); if (argc<1) return badusage(exename); @@ -767,21 +767,29 @@ int main(int argc, const char** argv) benchNb = readU32FromChar(&argument); break; - /* Modify Nb Iterations */ - case 'i': + /* Select compression level to use */ + case 'l': argument++; - g_nbIterations = readU32FromChar(&argument); + cLevel = (int)readU32FromChar(&argument); + cparams = ZSTD_getCParams(cLevel, 0, 0); break; /* Select compressibility of synthetic sample */ case 'P': argument++; - g_compressibility = (double)readU32FromChar(&argument) / 100.; + compressibility = (double)readU32FromChar(&argument) / 100.; break; - case 'l': + + /* Select size of synthetic sample */ + case 'B': argument++; - cLevel = (int)readU32FromChar(&argument); - cparams = ZSTD_getCParams(cLevel, 0, 0); + sampleSize = (size_t)readU32FromChar(&argument); + break; + + /* Modify Nb Iterations */ + case 'i': + argument++; + g_nbIterations = readU32FromChar(&argument); break; /* Unknown command */ @@ -798,7 +806,7 @@ int main(int argc, const char** argv) if (filenamesStart==0) /* no input file */ - result = benchSample(benchNb, cLevel, cparams); + result = benchSample(benchNb, sampleSize, compressibility, cLevel, cparams); else result = benchFiles(benchNb, argv+filenamesStart, argc-filenamesStart, cLevel, cparams); From eb6b1990384364f6a3aa50bd695f87da035e202e Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 28 May 2019 16:50:49 -0700 Subject: [PATCH 2/7] fullbench: added streaming with fresh CCtx scenario worst case situation, where context must be recreated every time but without knowledge of the input size thus sizing the context for some large input. --- tests/fullbench.c | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/tests/fullbench.c b/tests/fullbench.c index b996fc5cd..3340e9b53 100644 --- a/tests/fullbench.c +++ b/tests/fullbench.c @@ -160,6 +160,19 @@ local_ZSTD_compressStream(const void* src, size_t srcSize, return buffOut.pos; } +static size_t +local_ZSTD_compressStream_freshCCtx(const void* src, size_t srcSize, + void* dst, size_t dstCapacity, + void* buff2) +{ + ZSTD_CCtx* const cctx = ZSTD_createCCtx(); + assert(cctx != NULL); + + return local_ZSTD_compressStream(src, srcSize, dst, dstCapacity, buff2); + + ZSTD_freeCCtx(cctx); +} + static size_t local_ZSTD_compress_generic_end(const void* src, size_t srcSize, void* dst, size_t dstCapacity, @@ -354,6 +367,9 @@ static int benchMem(unsigned benchNb, case 42: benchFunction = local_ZSTD_decompressStream; benchName = "decompressStream"; break; + case 43: + benchFunction = local_ZSTD_compressStream_freshCCtx; benchName = "compressStream_freshCCtx"; + break; case 51: benchFunction = local_ZSTD_compress_generic_continue; benchName = "compress_generic, continue"; break; From c63081623f76c909e7bd2f1bd523c9a1fe9495f3 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 28 May 2019 17:27:52 -0700 Subject: [PATCH 3/7] fullbench 43: ensure context is freed after each usage --- tests/fullbench.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/fullbench.c b/tests/fullbench.c index 3340e9b53..cd6e74546 100644 --- a/tests/fullbench.c +++ b/tests/fullbench.c @@ -166,11 +166,14 @@ local_ZSTD_compressStream_freshCCtx(const void* src, size_t srcSize, void* buff2) { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); + size_t r; assert(cctx != NULL); - return local_ZSTD_compressStream(src, srcSize, dst, dstCapacity, buff2); + r = local_ZSTD_compressStream(src, srcSize, dst, dstCapacity, buff2); ZSTD_freeCCtx(cctx); + + return r; } static size_t From ed38b645db5f28dac223376beb8c3b728c208ee6 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 29 May 2019 15:26:06 -0700 Subject: [PATCH 4/7] fullbench: pass proper parameters in scenario 43 --- lib/compress/zstd_compress.c | 14 +++++++------- lib/compress/zstd_opt.c | 2 +- tests/fullbench.c | 3 +++ tests/fuzzer.c | 8 ++++---- 4 files changed, 15 insertions(+), 12 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 0e522a711..45f336796 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -930,12 +930,12 @@ size_t ZSTD_CCtx_reset(ZSTD_CCtx* cctx, ZSTD_ResetDirective reset) @return : 0, or an error code if one value is beyond authorized range */ size_t ZSTD_checkCParams(ZSTD_compressionParameters cParams) { - BOUNDCHECK(ZSTD_c_windowLog, cParams.windowLog); - BOUNDCHECK(ZSTD_c_chainLog, cParams.chainLog); - BOUNDCHECK(ZSTD_c_hashLog, cParams.hashLog); - BOUNDCHECK(ZSTD_c_searchLog, cParams.searchLog); - BOUNDCHECK(ZSTD_c_minMatch, cParams.minMatch); - BOUNDCHECK(ZSTD_c_targetLength,cParams.targetLength); + BOUNDCHECK(ZSTD_c_windowLog, (int)cParams.windowLog); + BOUNDCHECK(ZSTD_c_chainLog, (int)cParams.chainLog); + BOUNDCHECK(ZSTD_c_hashLog, (int)cParams.hashLog); + BOUNDCHECK(ZSTD_c_searchLog, (int)cParams.searchLog); + BOUNDCHECK(ZSTD_c_minMatch, (int)cParams.minMatch); + BOUNDCHECK(ZSTD_c_targetLength,(int)cParams.targetLength); BOUNDCHECK(ZSTD_c_strategy, cParams.strategy); return 0; } @@ -951,7 +951,7 @@ ZSTD_clampCParams(ZSTD_compressionParameters cParams) if ((int)valbounds.upperBound) val=(type)bounds.upperBound; \ } -# define CLAMP(cParam, val) CLAMP_TYPE(cParam, val, int) +# define CLAMP(cParam, val) CLAMP_TYPE(cParam, val, unsigned) CLAMP(ZSTD_c_windowLog, cParams.windowLog); CLAMP(ZSTD_c_chainLog, cParams.chainLog); CLAMP(ZSTD_c_hashLog, cParams.hashLog); diff --git a/lib/compress/zstd_opt.c b/lib/compress/zstd_opt.c index 3f2f2b7ba..534bae432 100644 --- a/lib/compress/zstd_opt.c +++ b/lib/compress/zstd_opt.c @@ -1094,7 +1094,7 @@ _shortestPath: /* cur, last_pos, best_mlen, best_off have to be set */ } /* while (ip < ilimit) */ /* Return the last literals size */ - return iend - anchor; + return (size_t)(iend - anchor); } diff --git a/tests/fullbench.c b/tests/fullbench.c index cd6e74546..9e0770b86 100644 --- a/tests/fullbench.c +++ b/tests/fullbench.c @@ -499,6 +499,9 @@ static int benchMem(unsigned benchNb, case 42 : g_cSize = ZSTD_compress(buff2, dstBuffSize, src, srcSize, cLevel); break; + case 43 : + buff2 = &cparams; + break; /* test functions */ /* convention: test functions have ID > 100 */ diff --git a/tests/fuzzer.c b/tests/fuzzer.c index 1fe488a25..2b6bfff6a 100644 --- a/tests/fuzzer.c +++ b/tests/fuzzer.c @@ -2464,7 +2464,7 @@ static unsigned readU32FromChar(const char** stringPtr) * If yes, @return 1 and advances *stringPtr to the position which immediately follows longCommand. * @return 0 and doesn't modify *stringPtr otherwise. */ -static unsigned longCommandWArg(const char** stringPtr, const char* longCommand) +static int longCommandWArg(const char** stringPtr, const char* longCommand) { size_t const comSize = strlen(longCommand); int const result = !strncmp(*stringPtr, longCommand, comSize); @@ -2524,7 +2524,7 @@ int main(int argc, const char** argv) case 'i': argument++; maxDuration = 0; - nbTests = readU32FromChar(&argument); + nbTests = (int)readU32FromChar(&argument); break; case 'T': @@ -2544,12 +2544,12 @@ int main(int argc, const char** argv) case 't': argument++; - testNb = readU32FromChar(&argument); + testNb = (int)readU32FromChar(&argument); break; case 'P': /* compressibility % */ argument++; - proba = readU32FromChar(&argument); + proba = (int)readU32FromChar(&argument); if (proba>100) proba = 100; break; From 904d4da239d54d0cc81721c9eedd1bc98d9a69a1 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 29 May 2019 16:08:49 -0700 Subject: [PATCH 5/7] fullbench : minor refactoring, for readability --- tests/fullbench.c | 123 ++++++++++++++++++++++++---------------------- 1 file changed, 64 insertions(+), 59 deletions(-) diff --git a/tests/fullbench.c b/tests/fullbench.c index 9e0770b86..3c470315e 100644 --- a/tests/fullbench.c +++ b/tests/fullbench.c @@ -99,12 +99,12 @@ static ZSTD_CCtx* g_zcc = NULL; static size_t local_ZSTD_compress(const void* src, size_t srcSize, void* dst, size_t dstSize, - void* buff2) + void* payload) { ZSTD_parameters p; ZSTD_frameParameters f = { 1 /* contentSizeHeader*/, 0, 0 }; p.fParams = f; - p.cParams = *(ZSTD_compressionParameters*)buff2; + p.cParams = *(ZSTD_compressionParameters*)payload; return ZSTD_compress_advanced (g_zcc, dst, dstSize, src, srcSize, NULL ,0, p); //return ZSTD_compress(dst, dstSize, src, srcSize, cLevel); } @@ -125,7 +125,7 @@ extern size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* ctx, const void* src, size_t s static size_t local_ZSTD_decodeLiteralsBlock(const void* src, size_t srcSize, void* dst, size_t dstSize, void* buff2) { (void)src; (void)srcSize; (void)dst; (void)dstSize; - return ZSTD_decodeLiteralsBlock((ZSTD_DCtx*)g_zdc, buff2, g_cSize); + return ZSTD_decodeLiteralsBlock(g_zdc, buff2, g_cSize); } static size_t local_ZSTD_decodeSeqHeaders(const void* src, size_t srcSize, void* dst, size_t dstSize, void* buff2) @@ -140,14 +140,14 @@ static ZSTD_CStream* g_cstream= NULL; static size_t local_ZSTD_compressStream(const void* src, size_t srcSize, void* dst, size_t dstCapacity, - void* buff2) + void* payload) { ZSTD_outBuffer buffOut; ZSTD_inBuffer buffIn; ZSTD_parameters p; ZSTD_frameParameters f = {1 /* contentSizeHeader*/, 0, 0}; p.fParams = f; - p.cParams = *(ZSTD_compressionParameters*)buff2; + p.cParams = *(ZSTD_compressionParameters*)payload; ZSTD_initCStream_advanced(g_cstream, NULL, 0, p, ZSTD_CONTENTSIZE_UNKNOWN); buffOut.dst = dst; buffOut.size = dstCapacity; @@ -163,13 +163,13 @@ local_ZSTD_compressStream(const void* src, size_t srcSize, static size_t local_ZSTD_compressStream_freshCCtx(const void* src, size_t srcSize, void* dst, size_t dstCapacity, - void* buff2) + void* payload) { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); size_t r; assert(cctx != NULL); - r = local_ZSTD_compressStream(src, srcSize, dst, dstCapacity, buff2); + r = local_ZSTD_compressStream(src, srcSize, dst, dstCapacity, payload); ZSTD_freeCCtx(cctx); @@ -179,20 +179,20 @@ local_ZSTD_compressStream_freshCCtx(const void* src, size_t srcSize, static size_t local_ZSTD_compress_generic_end(const void* src, size_t srcSize, void* dst, size_t dstCapacity, - void* buff2) + void* payload) { - (void)buff2; + (void)payload; return ZSTD_compress2(g_cstream, dst, dstCapacity, src, srcSize); } static size_t local_ZSTD_compress_generic_continue(const void* src, size_t srcSize, void* dst, size_t dstCapacity, - void* buff2) + void* payload) { ZSTD_outBuffer buffOut; ZSTD_inBuffer buffIn; - (void)buff2; + (void)payload; buffOut.dst = dst; buffOut.size = dstCapacity; buffOut.pos = 0; @@ -207,9 +207,9 @@ local_ZSTD_compress_generic_continue(const void* src, size_t srcSize, static size_t local_ZSTD_compress_generic_T2_end(const void* src, size_t srcSize, void* dst, size_t dstCapacity, - void* buff2) + void* payload) { - (void)buff2; + (void)payload; ZSTD_CCtx_setParameter(g_cstream, ZSTD_c_nbWorkers, 2); return ZSTD_compress2(g_cstream, dst, dstCapacity, src, srcSize); } @@ -217,11 +217,11 @@ local_ZSTD_compress_generic_T2_end(const void* src, size_t srcSize, static size_t local_ZSTD_compress_generic_T2_continue(const void* src, size_t srcSize, void* dst, size_t dstCapacity, - void* buff2) + void* payload) { ZSTD_outBuffer buffOut; ZSTD_inBuffer buffIn; - (void)buff2; + (void)payload; ZSTD_CCtx_setParameter(g_cstream, ZSTD_c_nbWorkers, 2); buffOut.dst = dst; buffOut.size = dstCapacity; @@ -257,27 +257,28 @@ local_ZSTD_decompressStream(const void* src, size_t srcSize, #ifndef ZSTD_DLL_IMPORT static size_t local_ZSTD_compressContinue(const void* src, size_t srcSize, void* dst, size_t dstCapacity, - void* buff2) + void* payload) { ZSTD_parameters p; ZSTD_frameParameters f = { 1 /* contentSizeHeader*/, 0, 0 }; p.fParams = f; - p.cParams = *(ZSTD_compressionParameters*)buff2; + p.cParams = *(ZSTD_compressionParameters*)payload; ZSTD_compressBegin_advanced(g_zcc, NULL, 0, p, srcSize); return ZSTD_compressEnd(g_zcc, dst, dstCapacity, src, srcSize); } #define FIRST_BLOCK_SIZE 8 -static size_t local_ZSTD_compressContinue_extDict(const void* src, size_t srcSize, - void* dst, size_t dstCapacity, - void* buff2) +static size_t +local_ZSTD_compressContinue_extDict(const void* src, size_t srcSize, + void* dst, size_t dstCapacity, + void* payload) { BYTE firstBlockBuf[FIRST_BLOCK_SIZE]; ZSTD_parameters p; - ZSTD_frameParameters f = { 1, 0, 0 }; + ZSTD_frameParameters const f = { 1, 0, 0 }; p.fParams = f; - p.cParams = *(ZSTD_compressionParameters*)buff2; + p.cParams = *(ZSTD_compressionParameters*)payload; ZSTD_compressBegin_advanced(g_zcc, NULL, 0, p, srcSize); memcpy(firstBlockBuf, src, FIRST_BLOCK_SIZE); @@ -333,7 +334,7 @@ static int benchMem(unsigned benchNb, size_t dstBuffSize = ZSTD_compressBound(srcSize); BYTE* dstBuff; void* dstBuff2; - void* buff2; + void* payload; const char* benchName; BMK_benchFn_t benchFunction; int errorcode = 0; @@ -397,7 +398,7 @@ static int benchMem(unsigned benchNb, free(dstBuff); free(dstBuff2); return 12; } - buff2 = dstBuff2; + payload = dstBuff2; if (g_zcc==NULL) g_zcc = ZSTD_createCCtx(); if (g_zdc==NULL) g_zdc = ZSTD_createDCtx(); if (g_cstream==NULL) g_cstream = ZSTD_createCStream(); @@ -430,62 +431,66 @@ static int benchMem(unsigned benchNb, switch(benchNb) { case 1: - buff2 = &cparams; + payload = &cparams; break; case 2: - g_cSize = ZSTD_compress(buff2, dstBuffSize, src, srcSize, cLevel); + g_cSize = ZSTD_compress(dstBuff2, dstBuffSize, src, srcSize, cLevel); break; #ifndef ZSTD_DLL_IMPORT case 11: - buff2 = &cparams; + payload = &cparams; break; case 12: - buff2 = &cparams; + payload = &cparams; break; case 13 : - g_cSize = ZSTD_compress(buff2, dstBuffSize, src, srcSize, cLevel); + g_cSize = ZSTD_compress(dstBuff2, dstBuffSize, src, srcSize, cLevel); break; - case 31: /* ZSTD_decodeLiteralsBlock */ - { blockProperties_t bp; - ZSTD_frameHeader zfp; - size_t frameHeaderSize, skippedSize; + case 31: /* ZSTD_decodeLiteralsBlock : starts literals block in dstBuff2 */ + { size_t frameHeaderSize; g_cSize = ZSTD_compress(dstBuff, dstBuffSize, src, srcSize, cLevel); - frameHeaderSize = ZSTD_getFrameHeader(&zfp, dstBuff, ZSTD_FRAMEHEADERSIZE_MIN); - if (frameHeaderSize==0) frameHeaderSize = ZSTD_FRAMEHEADERSIZE_MIN; - ZSTD_getcBlockSize(dstBuff+frameHeaderSize, dstBuffSize, &bp); /* Get 1st block type */ - if (bp.blockType != bt_compressed) { - DISPLAY("ZSTD_decodeLiteralsBlock : impossible to test on this sample (not compressible)\n"); - goto _cleanOut; + frameHeaderSize = ZSTD_frameHeaderSize(dstBuff, ZSTD_FRAMEHEADERSIZE_PREFIX); + assert(!ZSTD_isError(frameHeaderSize)); + /* check block is compressible, hence contains a literals section */ + { blockProperties_t bp; + ZSTD_getcBlockSize(dstBuff+frameHeaderSize, dstBuffSize, &bp); /* Get 1st block type */ + if (bp.blockType != bt_compressed) { + DISPLAY("ZSTD_decodeLiteralsBlock : impossible to test on this sample (not compressible)\n"); + goto _cleanOut; + } } + { size_t const skippedSize = frameHeaderSize + ZSTD_blockHeaderSize; + memcpy(dstBuff2, dstBuff+skippedSize, g_cSize-skippedSize); } - skippedSize = frameHeaderSize + ZSTD_blockHeaderSize; - memcpy(buff2, dstBuff+skippedSize, g_cSize-skippedSize); srcSize = srcSize > 128 KB ? 128 KB : srcSize; /* speed relative to block */ ZSTD_decompressBegin(g_zdc); break; } case 32: /* ZSTD_decodeSeqHeaders */ { blockProperties_t bp; - ZSTD_frameHeader zfp; const BYTE* ip = dstBuff; const BYTE* iend; - size_t frameHeaderSize, cBlockSize; - ZSTD_compress(dstBuff, dstBuffSize, src, srcSize, cLevel); /* it would be better to use direct block compression here */ - g_cSize = ZSTD_compress(dstBuff, dstBuffSize, src, srcSize, cLevel); - frameHeaderSize = ZSTD_getFrameHeader(&zfp, dstBuff, ZSTD_FRAMEHEADERSIZE_MIN); - if (frameHeaderSize==0) frameHeaderSize = ZSTD_FRAMEHEADERSIZE_MIN; - ip += frameHeaderSize; /* Skip frame Header */ - cBlockSize = ZSTD_getcBlockSize(ip, dstBuffSize, &bp); /* Get 1st block type */ - if (bp.blockType != bt_compressed) { - DISPLAY("ZSTD_decodeSeqHeaders : impossible to test on this sample (not compressible)\n"); - goto _cleanOut; + { size_t const cSize = ZSTD_compress(dstBuff, dstBuffSize, src, srcSize, cLevel); + assert(cSize > ZSTD_FRAMEHEADERSIZE_PREFIX); } - iend = ip + ZSTD_blockHeaderSize + cBlockSize; /* End of first block */ - ip += ZSTD_blockHeaderSize; /* skip block header */ + /* Skip frame Header */ + { size_t const frameHeaderSize = ZSTD_frameHeaderSize(dstBuff, ZSTD_FRAMEHEADERSIZE_PREFIX); + assert(!ZSTD_isError(frameHeaderSize)); + ip += frameHeaderSize; + } + /* Find end of block */ + { size_t const cBlockSize = ZSTD_getcBlockSize(ip, dstBuffSize, &bp); /* Get 1st block type */ + if (bp.blockType != bt_compressed) { + DISPLAY("ZSTD_decodeSeqHeaders : impossible to test on this sample (not compressible)\n"); + goto _cleanOut; + } + iend = ip + ZSTD_blockHeaderSize + cBlockSize; /* End of first block */ + } + ip += ZSTD_blockHeaderSize; /* skip block header */ ZSTD_decompressBegin(g_zdc); assert(iend > ip); ip += ZSTD_decodeLiteralsBlock(g_zdc, ip, (size_t)(iend-ip)); /* skip literal segment */ g_cSize = (size_t)(iend-ip); - memcpy(buff2, ip, g_cSize); /* copy rest of block (it starts by SeqHeader) */ + memcpy(dstBuff2, ip, g_cSize); /* copy rest of block (it starts by SeqHeader) */ srcSize = srcSize > 128 KB ? 128 KB : srcSize; /* speed relative to block */ break; } @@ -494,13 +499,13 @@ static int benchMem(unsigned benchNb, goto _cleanOut; #endif case 41 : - buff2 = &cparams; + payload = &cparams; break; case 42 : - g_cSize = ZSTD_compress(buff2, dstBuffSize, src, srcSize, cLevel); + g_cSize = ZSTD_compress(payload, dstBuffSize, src, srcSize, cLevel); break; case 43 : - buff2 = &cparams; + payload = &cparams; break; /* test functions */ @@ -522,7 +527,7 @@ static int benchMem(unsigned benchNb, assert(tfs != NULL); bp.benchFn = benchFunction; - bp.benchPayload = buff2; + bp.benchPayload = payload; bp.initFn = NULL; bp.initPayload = NULL; bp.errorFn = ZSTD_isError; From ab3625cffda9be5773472d42db0f8977927ce74f Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 29 May 2019 16:18:22 -0700 Subject: [PATCH 6/7] define CONTROL, to ensure check cannot be disabled assert() can be disabled with NDEBUG --- tests/fullbench.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/fullbench.c b/tests/fullbench.c index 3c470315e..fcae1f16a 100644 --- a/tests/fullbench.c +++ b/tests/fullbench.c @@ -15,7 +15,6 @@ #include "util.h" /* Compiler options, UTIL_GetFileSize */ #include /* malloc */ #include /* fprintf, fopen, ftello64 */ -#include /* assert */ #include "timefn.h" /* UTIL_clockSpanNano, UTIL_getTime */ #include "mem.h" /* U32 */ @@ -61,6 +60,7 @@ static const size_t k_sampleSize_default = 10000000; **************************************/ #define DISPLAY(...) fprintf(stderr, __VA_ARGS__) +#define CONTROL(c) { if (!(c)) { abort(); } } /* like assert(), but cannot be disabled */ /*_************************************ * Benchmark Parameters @@ -450,7 +450,7 @@ static int benchMem(unsigned benchNb, { size_t frameHeaderSize; g_cSize = ZSTD_compress(dstBuff, dstBuffSize, src, srcSize, cLevel); frameHeaderSize = ZSTD_frameHeaderSize(dstBuff, ZSTD_FRAMEHEADERSIZE_PREFIX); - assert(!ZSTD_isError(frameHeaderSize)); + CONTROL(!ZSTD_isError(frameHeaderSize)); /* check block is compressible, hence contains a literals section */ { blockProperties_t bp; ZSTD_getcBlockSize(dstBuff+frameHeaderSize, dstBuffSize, &bp); /* Get 1st block type */ @@ -470,11 +470,11 @@ static int benchMem(unsigned benchNb, const BYTE* ip = dstBuff; const BYTE* iend; { size_t const cSize = ZSTD_compress(dstBuff, dstBuffSize, src, srcSize, cLevel); - assert(cSize > ZSTD_FRAMEHEADERSIZE_PREFIX); + CONTROL(cSize > ZSTD_FRAMEHEADERSIZE_PREFIX); } /* Skip frame Header */ { size_t const frameHeaderSize = ZSTD_frameHeaderSize(dstBuff, ZSTD_FRAMEHEADERSIZE_PREFIX); - assert(!ZSTD_isError(frameHeaderSize)); + CONTROL(!ZSTD_isError(frameHeaderSize)); ip += frameHeaderSize; } /* Find end of block */ @@ -487,7 +487,7 @@ static int benchMem(unsigned benchNb, } ip += ZSTD_blockHeaderSize; /* skip block header */ ZSTD_decompressBegin(g_zdc); - assert(iend > ip); + CONTROL(iend > ip); ip += ZSTD_decodeLiteralsBlock(g_zdc, ip, (size_t)(iend-ip)); /* skip literal segment */ g_cSize = (size_t)(iend-ip); memcpy(dstBuff2, ip, g_cSize); /* copy rest of block (it starts by SeqHeader) */ @@ -524,7 +524,7 @@ static int benchMem(unsigned benchNb, BMK_runTime_t bestResult; bestResult.sumOfReturn = 0; bestResult.nanoSecPerRun = (double)TIMELOOP_NANOSEC * 2000000000; /* hopefully large enough : must be larger than any potential measurement */ - assert(tfs != NULL); + CONTROL(tfs != NULL); bp.benchFn = benchFunction; bp.benchPayload = payload; @@ -751,7 +751,7 @@ int main(int argc, const char** argv) for (argNb=1; argNb /* malloc */ #include /* fprintf, fopen, ftello64 */ +#include #include "timefn.h" /* UTIL_clockSpanNano, UTIL_getTime */ #include "mem.h" /* U32 */ @@ -30,8 +31,8 @@ #include "zstd.h" /* ZSTD_versionString */ #include "util.h" /* time functions */ #include "datagen.h" -#include "benchfn.h" /* CustomBench*/ -#include "benchzstd.h" /* MB_UNIT */ +#include "benchfn.h" /* CustomBench */ +#include "benchzstd.h" /* MB_UNIT */ /*_************************************ @@ -50,7 +51,7 @@ #define DEFAULT_CLEVEL 1 #define COMPRESSIBILITY_DEFAULT 0.50 -static const size_t k_sampleSize_default = 10000000; +static const size_t kSampleSizeDefault = 10000000; #define TIMELOOP_NANOSEC (1*1000000000ULL) /* 1 second */ @@ -723,7 +724,7 @@ static int usage_advanced(const char* exename) DISPLAY( " -l# : benchmark functions at that compression level (default : %i)\n", DEFAULT_CLEVEL); DISPLAY( " --zstd : custom parameter selection. Format same as zstdcli \n"); DISPLAY( " -P# : sample compressibility (default : %.1f%%)\n", COMPRESSIBILITY_DEFAULT * 100); - DISPLAY( " -B# : sample size (default : %u)\n", (unsigned)k_sampleSize_default); + DISPLAY( " -B# : sample size (default : %u)\n", (unsigned)kSampleSizeDefault); DISPLAY( " -i# : iteration loops [1-9](default : %i)\n", NBLOOPS); return 0; } @@ -743,7 +744,7 @@ int main(int argc, const char** argv) U32 benchNb = 0, main_pause = 0; int cLevel = DEFAULT_CLEVEL; ZSTD_compressionParameters cparams = ZSTD_getCParams(cLevel, 0, 0); - size_t sampleSize = k_sampleSize_default; + size_t sampleSize = kSampleSizeDefault; double compressibility = COMPRESSIBILITY_DEFAULT; DISPLAY(WELCOME_MESSAGE);