From 678272515515afc6fe99975a8c716ce27907fc7e Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Sun, 26 Aug 2018 19:29:12 -0700 Subject: [PATCH 01/15] first sketch for largeNbDicts test program --- contrib/largeNbDicts/Makefile | 30 +++ contrib/largeNbDicts/largeNbDicts | Bin 0 -> 13626 bytes contrib/largeNbDicts/largeNbDicts.c | 289 ++++++++++++++++++++++++++++ lib/Makefile | 2 +- lib/dictBuilder/zdict.h | 3 +- programs/bench.h | 2 +- 6 files changed, 323 insertions(+), 3 deletions(-) create mode 100644 contrib/largeNbDicts/Makefile create mode 100755 contrib/largeNbDicts/largeNbDicts create mode 100644 contrib/largeNbDicts/largeNbDicts.c diff --git a/contrib/largeNbDicts/Makefile b/contrib/largeNbDicts/Makefile new file mode 100644 index 000000000..082f0102a --- /dev/null +++ b/contrib/largeNbDicts/Makefile @@ -0,0 +1,30 @@ +# ################################################################ +# Copyright (c) 2018-present, Yann Collet, Facebook, Inc. +# All rights reserved. +# +# This source code is licensed under both the BSD-style license (found in the +# LICENSE file in the root directory of this source tree) and the GPLv2 (found +# in the COPYING file in the root directory of this source tree). +# ################################################################ + + +CPPFLAGS+= -I../../lib -I../../lib/common -I../../lib/dictBuilder -I../../programs + +CFLAGS ?= -O3 +DEBUGFLAGS= -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \ + -Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \ + -Wstrict-prototypes -Wundef -Wpointer-arith -Wformat-security \ + -Wvla -Wformat=2 -Winit-self -Wfloat-equal -Wwrite-strings \ + -Wredundant-decls +CFLAGS += $(DEBUGFLAGS) $(MOREFLAGS) + + +default: largeNbDicts + +largeNbDicts: LDFLAGS += -lzstd +largeNbDicts: largeNbDicts.c + $(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@ + + +clean: + $(RM) largeNbDicts diff --git a/contrib/largeNbDicts/largeNbDicts b/contrib/largeNbDicts/largeNbDicts new file mode 100755 index 0000000000000000000000000000000000000000..40416f050fa594cb085444ebb1ca433629d59318 GIT binary patch literal 13626 zcmX^A>+L^w1_nlE28ISE1_lNJ1_p)+tPBjT3yFff4lEC}Tc3@i){$lUn&;*!#&Vz^LzJgRwNG7$69pekT|D3^f)Y97LU zAoJogQgaGYi?FzF57fL}Py;}GP`E?6SlpMKpI40VFuMC1WFhX;fEobeqxcsp4x{4Z zlZ#7=GV{`*0_f(gfSSh)brgsXRSKq{!eBOth>tJLE6>bJiO1BUlJRC_u~sCs2?GG=0GMpy+01U|;}YkaT=}dOjl5 zaGS>gQVhZh2)!UKC_V*23@DC|&&!D~uFOr!&xtQ6DPo9^M|B?u)P11*1JVNG!`uSm zgUk~GF`zg;J|_{Mc@j|bI-u%7d}Q-L{uP1BfhbV8LGr1Or=Pd0izh6P8K8xm0Z26i zgAFKCF)%QI%mL*Wh$sVt5(7g6Scw4x11M}b3>X+-X$z#rz<_}vfRTZr!H|I=fq{Vm z6qgbV3=A&}85ltHA7cgv2Sx@4ZUzPhSe%3OfiTEQ5Ece;aNrDM1_o6ua(oO7;1Y?4 zfq_B4w75t=Co@Sur7|Z4s#cl-V*ircn~p3Gv|2K0+m5F;UQ^eoK+7}+E(QiJ1}zv5 zs!M}`p#kJ-1{tUTlmgk}05uWhLl$OG7%@U^&QDIv(a$d^(XT2lNdY?~Co>7e)-%*g zsm#etVgTv4K-O>cgas52aF^#~CIwd(m!#(EIYG66ZB~Q2?|~sC^fjS;Q0Rgcj1r?E zFd71*Aut*OqaiRF0;3@?8UmvsFd71*Aut*OqaiRF0yGbSP>;^1j^U1Bj-ier!5+2X+Pq&(5PD zrYV?t#JBZriMwy>lM*LS%X|Dyri=^>9=)yt9^DKc-L)q?7?1mO{`2^Mj=$wK0|SF+ zZ}`8?kIfI+J(_>97dseU@@PHD-@1{3f#KyD1_lP7&i5}av4QC4FHW&BFnDykUhp{X z03MF=IPL%%%3<(04sr{_%e|m6UCncbmnL|2x?E-O?3B6U!FU72=J4!{xxxYE2!J>O zP>uwMBLU?ofH(?Jjs}RM0p%EgI0jIT1&Cwe!Fa=?`4ERk^RXYL{T{6+OB6hNO|E$M zhFoQs;L*wC(J2R3af}13>i@~oS00u>%Dp@^k9&8z{QcqGS@Y)y$TvQn8a|x~96p^D z0v^prB%&Q-9Ah2h9OGjTd-U4g26?&H^Z}USeZj`Su&Wm|Dz%Um6sER&z~ZI{!4&Ux zkn~iLbQ3EBLt0vz9>07GIA~DgxABbzC`vki`*iDG;$UF#?5<_-=rvu(#=u~Bz@zil zi%_uXTQB_k|KFqY{)-gQxcLG8))*!R2LA1>SN{F~ugdU&fx#9u!?T(ucg zwH|iW;!F$-KHXal{{R2)+5GQ6e+wH}yXg^j1_qF2msvo`>9}v}fBu%&jG&1B{^BlJ z>3O-?#M%$jkgK%NQ9L(meU|j-~M@v3h{Qj6a4o08G~$_{1N{ z8UhXo)4i+=3>z3<*s*}pUn|H6k4_gA0gvuhkl`NPy&%IqI(<|)JUUraJbL$5{Qv)d zA7}{o#iBp||9f<|f&@G|T~~Ot9^h}81oClrXoq9xb&xwedQF$JGBE4}sRxCB=!F+o zm>C%MgZMA{m_fmD%%}6SC;$5Upa8nS%)kIPCJSUtr|SlfPS*(@ov{l%I%8*ebh|Eq zy3KWhN4M_`WLG8p2L-{m7kpqh%>{8iT5t2WI55G}Kr<@?!!D4;h6i5w!yOR|afEN{ zH;>NP9sDgbKpCyuw*l-$pU&?-ov8meY5Xy) z31Au$-zlK5?KM5Y0*Y^s?$8S_yqFjmc7X<{!QOoKf&~)j7rc6H{(>C$!VzS4=TYC* zxBM+)j0_CEt^Z4m4R1r0-)00wpUYiPjFr@R{6Fkt`Mc<`Pv>ioZtE8pK)JB_Fo);= z^QG@W1>H-KKzHqh7aJHE7z}Uwbf+Hh>8?HD)A{|yIZ#&U-kR|rl$>gJyZ~hbkoY!; zmJKh~LKrJvECH$P{QqJOn123Z3L^u<%S)iF+-sWzO2NIR6G5Kn<;`MYVA!=AR9*yu zH22#60*jj#fhk^4)a|+fk~W4&-v*1D27@VHO=bp$T@OIgA`t04U~y9uFvTkgl70@7 z{t8Mhy|#0~;-;ctigz|h`aeke8Uq6ZxO@r=_5c-0ruLxR-p%LHYdeXVfngtLgdCDu zE`X|wZaa@&(+*}>T3E%vz_0_9f}$PcVh_X0yXOB)rJNq!bsR65e*gauGre2Rr`z-) zC@q6cH$gQWG^^zaHeCyDx(|x!|5f)eGcf#D-3KNQfXPE(@(7qb1}0B{$x~qR446Cz zCPDf3zv?A08VIgJbZ2Ra<5Th6x#533dGU@u>wFWYY@xr|9?8XB6E}6+C#d^sMc}Y$=`N`SE z3U;;%h75VBu8T1< zpg>nGRse-`USe*l0vAJ35;U}3Kr!i@pOcdc_5uUMlxUDyeo3+L@C46l!=|Z_Y0&%> zXqMZAfq?;p!=aKOHmE@6VPIkq0ngJhFgCC;FeoxGFNf+7C)cKF9jr{0ha*}Fgb&UJV4?GsN&$E5RkY9Ogy!?Br`X$Bo!q@o z;0yqd{_rz`Cj-IbNMZ~O4E&4?3>&0DtCB%uLJ+=yIE1eOl?Sbh1`QPHLHVF@CJ^76 z0jv(h1SJcGAT&N`w*(_t30Rt;0!_XPjlT$uzX^?h2#pUK;bw#^J_lQH4A3Sa12Z*YEOU|;~Pk_WBu7iC~z0L2$*d|w>wI|c`8n5V!Hk0ss3C&5!xJY=9Z z9-Q*xk<%lNA|O5rGxI~mkg`3p2qc?AL@={DL>!UH5qxBQu#6oaACHu!k;*ezzK%}= k=jiw}XkLy_1LtB8mC6vG2FbVa&pF literal 0 HcmV?d00001 diff --git a/contrib/largeNbDicts/largeNbDicts.c b/contrib/largeNbDicts/largeNbDicts.c new file mode 100644 index 000000000..749d9660d --- /dev/null +++ b/contrib/largeNbDicts/largeNbDicts.c @@ -0,0 +1,289 @@ +/* + * Copyright (c) 2018-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +/* largeNbDicts + * This is a benchmark test tool + * dedicated to the specific case of dictionary decompression + * using a very large nb of dictionaries + * thus generating many cache-misses. + * It's created in a bid to investigate performance and find optimizations. */ + + +/*--- Dependencies ---*/ + +#include /* size_t */ +#include /* malloc, free */ +#include /* printf */ +#include /* assert */ + +#include "util.h" +#define ZSTD_STATIC_LINKING_ONLY +#include "zstd.h" +#include "zdict.h" + + +/*--- Constants --- */ + +#define KB *(1<<10) +#define MB *(1<<20) + +#define BLOCKSIZE (4 KB) +#define DICTSIZE (4 KB) +#define COMP_LEVEL 3 + +#define DISPLAY_LEVEL_DEFAULT 3 + + +/*--- Display Macros ---*/ + +#define DISPLAY(...) fprintf(stdout, __VA_ARGS__) +#define DISPLAYLEVEL(l, ...) { if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); } } +static int g_displayLevel = DISPLAY_LEVEL_DEFAULT; /* 0 : no display, 1: errors, 2 : + result + interaction + warnings, 3 : + progression, 4 : + information */ + + +/*--- buffer_t ---*/ + +typedef struct { + void* ptr; + size_t size; + size_t capacity; +} buffer_t; + +static const buffer_t kBuffNull = { NULL, 0, 0 }; + + +static buffer_t fillBuffer_fromHandle(buffer_t buff, FILE* f) +{ + size_t const readSize = fread(buff.ptr, 1, buff.capacity, f); + buff.size = readSize; + return buff; +} + +static void freeBuffer(buffer_t buff) +{ + free(buff.ptr); +} + +/* @return : kBuffNull if any error */ +static buffer_t createBuffer_fromHandle(FILE* f, size_t bufferSize) +{ + void* const buffer = malloc(bufferSize); + if (buffer==NULL) return kBuffNull; + + { buffer_t buff = { buffer, 0, bufferSize }; + buff = fillBuffer_fromHandle(buff, f); + if (buff.size != buff.capacity) { + freeBuffer(buff); + return kBuffNull; + } + return buff; + } +} + +/* @return : kBuffNull if any error */ +static buffer_t createBuffer_fromFile(const char* fileName) +{ + U64 const fileSize = UTIL_getFileSize(fileName); + size_t const bufferSize = (size_t) fileSize; + + if (fileSize == UTIL_FILESIZE_UNKNOWN) return kBuffNull; + assert((U64)bufferSize == fileSize); /* check overflow */ + + { buffer_t buff; + FILE* const f = fopen(fileName, "rb"); + if (f == NULL) return kBuffNull; + + buff = createBuffer_fromHandle(f, bufferSize); + fclose(f); /* do nothing specific if fclose() fails */ + return buff; + } +} + + +/*--- buffer_collection_t ---*/ + +typedef struct { + void** buffers; + size_t* capacities; + size_t nbBuffers; +} buffer_collection_t; + +static const buffer_collection_t kNullCollection = { NULL, NULL, 0 }; + +static void freeCollection(buffer_collection_t collection) +{ + free(collection.buffers); + free(collection.capacities); +} + +/* returns .buffers=NULL if operation fails */ +buffer_collection_t splitBuffer(buffer_t srcBuffer, size_t blockSize) +{ + size_t const nbBlocks = (srcBuffer.size + (blockSize-1)) / blockSize; + + void** const buffers = malloc(nbBlocks * sizeof(void*)); + size_t* const capacities = malloc(nbBlocks * sizeof(size_t*)); + if ((buffers==NULL) || capacities==NULL) { + free(buffers); + free(capacities); + return kNullCollection; + } + + char* newBlockPtr = (char*)srcBuffer.ptr; + char* const srcEnd = newBlockPtr + srcBuffer.size; + assert(nbBlocks >= 1); + for (size_t blockNb = 0; blockNb < nbBlocks-1; blockNb++) { + buffers[blockNb] = newBlockPtr; + capacities[blockNb] = blockSize; + newBlockPtr += blockSize; + } + + /* last block */ + assert(newBlockPtr <= srcEnd); + size_t const lastBlockSize = (srcEnd - newBlockPtr); + buffers[nbBlocks-1] = newBlockPtr; + capacities[nbBlocks-1] = lastBlockSize; + + buffer_collection_t result; + result.buffers = buffers; + result.capacities = capacities; + result.nbBuffers = nbBlocks; + return result; +} + + + +/*--- ddict_collection_t ---*/ + +typedef struct { + ZSTD_DDict** ddicts; + size_t nbDDict; +} ddict_collection_t; + +static const ddict_collection_t kNullDDictCollection = { NULL, 0 }; + +static void freeDDictCollection(ddict_collection_t ddictc) +{ + for (size_t dictNb=0; dictNb < ddictc.nbDDict; dictNb++) { + ZSTD_freeDDict(ddictc.ddicts[dictNb]); + } + free(ddictc.ddicts); +} + +/* returns .buffers=NULL if operation fails */ +static ddict_collection_t createDDictCollection(const void* dictBuffer, size_t dictSize, size_t nbDDict) +{ + ZSTD_DDict** const ddicts = malloc(nbDDict * sizeof(ZSTD_DDict*)); + if (ddicts==NULL) return kNullDDictCollection; + for (size_t dictNb=0; dictNb < nbDDict; dictNb++) { + ddicts[dictNb] = ZSTD_createDDict(dictBuffer, dictSize); + assert(ddicts[dictNb] != NULL); + } + ddict_collection_t ddictc; + ddictc.ddicts = ddicts; + ddictc.nbDDict = nbDDict; + return ddictc; +} + + + +/*--- Benchmark --- */ + + +/* bench() : + * @return : 0 is success, 1+ otherwise */ +int bench(const char* fileName) +{ + int result = 0; + + DISPLAYLEVEL(3, "loading %s... \n", fileName); + buffer_t const srcBuffer = createBuffer_fromFile(fileName); + if (srcBuffer.ptr == NULL) { + DISPLAYLEVEL(1," error reading file %s \n", fileName); + return 1; + } + DISPLAYLEVEL(3, "created src buffer of size %.1f MB \n", + (double)(srcBuffer.size) / (1 MB)); + + buffer_collection_t const srcBlockBuffers = splitBuffer(srcBuffer, BLOCKSIZE); + assert(srcBlockBuffers.buffers != NULL); + unsigned const nbBlocks = (unsigned)srcBlockBuffers.nbBuffers; + DISPLAYLEVEL(3, "splitting input into %u blocks of max size %u bytes \n", + nbBlocks, BLOCKSIZE); + + size_t const dstBlockSize = ZSTD_compressBound(BLOCKSIZE); + size_t const dstBufferCapacity = nbBlocks * dstBlockSize; + void* const dstPtr = malloc(dstBufferCapacity); + assert(dstPtr != NULL); + buffer_t dstBuffer; + dstBuffer.ptr = dstPtr; + dstBuffer.capacity = dstBufferCapacity; + dstBuffer.size = dstBufferCapacity; + + buffer_collection_t const dstBlockBuffers = splitBuffer(dstBuffer, dstBlockSize); + assert(dstBlockBuffers.buffers != NULL); + + DISPLAYLEVEL(3, "creating dictionary, of target size %u bytes \n", DICTSIZE); + void* const dictBuffer = malloc(DICTSIZE); + if (dictBuffer == NULL) { result = 1; goto _cleanup; } + + size_t const dictSize = ZDICT_trainFromBuffer(dictBuffer, DICTSIZE, + srcBuffer.ptr, + srcBlockBuffers.capacities, + nbBlocks); + if (ZSTD_isError(dictSize)) { + DISPLAYLEVEL(1, "error creating dictionary \n"); + result = 1; + goto _cleanup; + } + + size_t const dictMem = ZSTD_estimateDDictSize(dictSize, ZSTD_dlm_byCopy); + size_t const allDictMem = dictMem * nbBlocks; + DISPLAYLEVEL(3, "generating %u dictionaries, using %.1f MB of memory \n", + nbBlocks, (double)allDictMem / (1 MB)); + + ZSTD_CDict* const cdict = ZSTD_createCDict(dictBuffer, dictSize, COMP_LEVEL); + do { + ddict_collection_t const dictionaries = createDDictCollection(dictBuffer, dictSize, nbBlocks); + assert(dictionaries.ddicts != NULL); + + freeDDictCollection(dictionaries); + } while(0); + ZSTD_freeCDict(cdict); + +_cleanup: + free(dictBuffer); + freeCollection(dstBlockBuffers); + freeBuffer(dstBuffer); + freeCollection(srcBlockBuffers); + freeBuffer(srcBuffer); + + return result; +} + + + + +/*--- Command Line ---*/ + +int bad_usage(const char* exeName) +{ + DISPLAY (" bad usage : \n"); + DISPLAY (" %s filename \n", exeName); + return 1; +} + +int main (int argc, const char** argv) +{ + const char* const exeName = argv[0]; + + if (argc != 2) return bad_usage(exeName); + return bench(argv[1]); +} diff --git a/lib/Makefile b/lib/Makefile index 01689c6d5..cf8e45b0f 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -23,7 +23,7 @@ ifeq ($(OS),Windows_NT) # MinGW assumed CPPFLAGS += -D__USE_MINGW_ANSI_STDIO # compatibility with %zu formatting endif CFLAGS ?= -O3 -DEBUGFLAGS = -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \ +DEBUGFLAGS= -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \ -Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \ -Wstrict-prototypes -Wundef -Wpointer-arith -Wformat-security \ -Wvla -Wformat=2 -Winit-self -Wfloat-equal -Wwrite-strings \ diff --git a/lib/dictBuilder/zdict.h b/lib/dictBuilder/zdict.h index 4094669d1..3b3a6527c 100644 --- a/lib/dictBuilder/zdict.h +++ b/lib/dictBuilder/zdict.h @@ -52,7 +52,8 @@ extern "C" { * It's recommended that total size of all samples be about ~x100 times the target size of dictionary. */ ZDICTLIB_API size_t ZDICT_trainFromBuffer(void* dictBuffer, size_t dictBufferCapacity, - const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples); + const void* samplesBuffer, + const size_t* samplesSizes, unsigned nbSamples); /*====== Helper functions ======*/ diff --git a/programs/bench.h b/programs/bench.h index f6a53fc6a..184cc3e00 100644 --- a/programs/bench.h +++ b/programs/bench.h @@ -233,7 +233,7 @@ typedef size_t (*BMK_initFn_t)(void* initPayload); * srcSizes - an array of the sizes of above buffers * dstBuffers - an array of buffers to be written into by benchFn * dstCapacities - an array of the capacities of above buffers - * blockResults - store the return value of benchFn for each block. Optional. Use NULL if this result is not requested. + * blockResults - Optional: store the return value of benchFn for each block. Use NULL if this result is not requested. * nbLoops - defines number of times benchFn is run. * @return: a variant, which express either an error, or can generate a valid BMK_runTime_t result. * Use BMK_isSuccessful_runOutcome() to check if function was successful. From 274b60e6e6571ccce5982e36ac82eabe606c1364 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Mon, 27 Aug 2018 17:08:44 -0700 Subject: [PATCH 02/15] largeNbDicts can compress and compare dict vs noDict --- contrib/largeNbDicts/Makefile | 2 + contrib/largeNbDicts/largeNbDicts | Bin 13626 -> 14034 bytes contrib/largeNbDicts/largeNbDicts.c | 144 +++++++++++++++++++++------- 3 files changed, 114 insertions(+), 32 deletions(-) diff --git a/contrib/largeNbDicts/Makefile b/contrib/largeNbDicts/Makefile index 082f0102a..026d76f12 100644 --- a/contrib/largeNbDicts/Makefile +++ b/contrib/largeNbDicts/Makefile @@ -21,6 +21,8 @@ CFLAGS += $(DEBUGFLAGS) $(MOREFLAGS) default: largeNbDicts +all : largeNbDicts + largeNbDicts: LDFLAGS += -lzstd largeNbDicts: largeNbDicts.c $(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@ diff --git a/contrib/largeNbDicts/largeNbDicts b/contrib/largeNbDicts/largeNbDicts index 40416f050fa594cb085444ebb1ca433629d59318..c057a2b78aa551a2de831b6e304f8747a6ea3d0f 100755 GIT binary patch literal 14034 zcmX^A>+L^w1_nlE28ISE1_lNJ1_p)+tPBjT39lG3DNC=b(pEm9Ek>YyrMd?=TJ18N?^eIWDV zGg5O3Qj4&-k3||{-Xo|1AU-JEpiom1pLq#AoKE<%9XC?wcYHF)sn4 zodLv0Hv=jKragc@%1 zI6#U)Sb>27rWeEo#iuBU0mbq0c{%aLmAOgzIq?N0MGW!rsP5x{x(}3pKw3b2bo0bO z5>Ol;pOc8sJPD|IE1>E@d}Q-L{*{2rfhbV8LGr1Or=Pd0izh6P8K8xm0Z26i!xfMR z85kHq=791GM3jL+iGiU3ti*tU0TebI1`G_av;|UQV8FnzgOP#Zg&_k&1p@;EC@w+n zC@^ARkY`|E5HV$7IKarj0Lp#=APrD8Aax)sL1v0W#j#NdCJYR^SS0uu7{DbG4+8^( zera)$eokhReoAFd3RJB$0|Nud9d9k&nJ<5}jb4!?>6xKZt*!zs(-^oI7`Pa;U_7WU z4F-k=kggA);Dd^RD3C1``QgCH)Noua16I2VxS_Uqdf>B~L z1V%$(Gz3ONU^E0qLtr!nMniz25D4|?eCinP80Hx27!vH!{6@l~^Rq{1?FEl+R|$_! z*AqUyCCvW~FZpzS@c91Ov-7x5ugX!7g+86nUu@uJVDRib3Suq=GmrSTzAbV0ZGBSW zu$O zqXFd@fH($Fjs=Kg;lX&rqxlerNAs~CrTreQCrcDOdrhu*_J&+#nBdXL?0Cy8Qj&-C6VJ2go-*ofD16#^d3M0Hf}}rjGBBj2rRnj@ zw}68NC4K`uJI{M|Uh(Ms?AiIxqxH5&H|rlhP{>-8$a(adx^puy7#{HGy!B!qCn(&% zdvxA=v73Q`;dP8l=V6atQ#+6~p#G-effutm85s71*e_Ocf|CI#?%no*M0`5m`*c3? z>HO)~dEUnm?ad;kg6M!iH;L&`91J(OXV2_zL zf_=-&3bJ@D$bUj0<9cl$gT+mgz!dLKE(V5On?cf_Il$h>;{T9fkH$9>KUAFfj0MZw>kX|Gz532L=XP&^QTy>kN<{gzAG()s5Ixm*)9& zmu~QE{`bEm#iQ4>l?UYa`!6y%Kp}M8xAlLCuTSUq7v&rv@xvaiw@VcEf%;V*o%ek@ zKlyb2eBs5xz~E!~p@bLYGG`7323NyxhPQn>|G$s}8zQ6P+gYN*0U`umOL-o5Q2}QN z5CfbeJdd-0I(!TsjYmM91qCWNMPW@39*u8afHGp|H;>NWKHawOc^DWxyX`%CP4{!c zlEP+oP&01m8sFV_6~|KFp#6~c90;n8}4zhyNuSleg*y55Zc|Np1)C-F`N#RGo~ z?_4llbMh0v0P8|9TeBAw?w|N04|?>Pws0~qY+!uh53-}XwE%2~i;4iuf^M*R9-Tfa z93Gu4DjvOi75@MK4^Cq*oPUEI0uu1(bUgreh$Ay7G0!#l|NlS48=0I83_C$-%3sHL*)Oo;KwPlHo-u(U-1UM-_goE-alLyvz$(9a zbce3+=)CFE?YiMb;Gh5heY&@TWPDq{@wZ$6Yl=PL(^)&gr+cZu|NsAYfd)W)I=_SL zc)|)ca|hUh!%PeeKHYmAK!$fu^#D<@gwS1k0aWUChA!~v_C4U!?RvtaJ9dL-=gk)a zpy28}2r7147J@A6W_=6FP~8@39{hPHLGgH&je#MJKZf@zn65dQ#;?%|iN82d{4M2R zU;w*m1LKP>kbAmacYtFGY-#Nbkj32yi$Rvo23Z8Q)U)$Ge~Sl*+s#@EvbXcFN4GWH z>L9R{F}zV=8pG-a4p7o@>3sA;kd=YK@EfQWy34}A;K{%KDAbq(Y5Ym7++gi7tioWr z=D;WZNLFz+P?|QahZ^zg#W@xR2CrV5x1b{P#Z8bYoscwef{}p%RIqs(-cI9}cL5~` zaNy^H)v(+L2YwHzg!q5h$MSX21E0>{X~^ZTN4NEhAQlD&kLJT1p8wC5z6WLHY>+^A z?T!~lEMOB~L_q{DyaHe{93ou;kzNjw)&NQC zfTS%T(xDLP4v4e@NLn8xEeVk}gGgsWr1?S8Mj+{5pmfx0D-MzNf=K&-q(Nh;FK$Dm zQ$PhxucOqnUe+#H0GQ90-_}`4J7>%B<%%}PJ>9#f=JH;NxuO}Ycn!1fb$zTKEi^*^^>VT zsC4e;^XRqx#lpa_4>TgW3sil+07Zspx1C3?=?6%j=)Cpf7XxIRmLb|PF7_}iyujv# zJBEQ;yPcnXx{aIvGuEm4bXRk@8a`?M&r}lH{GX*%#G|`fz@yjnAPWP-i>nNv&?<58 z=$1VPa&9-bN3U%qssrYOihyowk6zP!umgNLKfS1AU|?|E19IPM0mE;vSv*KVu9NIFMB{NMn(pPmrWp+D2N5BO*1q>tUQpIIf#`4V!47?Q6N?*hy|*AGLk_o zH;`BXh-Cv})q_|@AXXoU1*&E<7K2!F|Ns97c?gCLoEaDlTwy#24H5&fVB#C>iVJcw zOPos6(o%~UauSQuQ~i=$GLuV+^^zI#lALn#le3Ez>}(Yb8S+xg!Q6n7A_W^;h2o-Q z*Sr*loczR;%)E4kl+5Ik%>2B>qDlqTVg)XSyEth5R&y5|9-o3dNaKsS2v4 z3Q3hEsc_|~Ad4Y}Fo4X1xJp6MR>3dS$A>{NDmcU?KC{@hs3^Zk1Ee50vno{+?gZ6h zJv}`IE{3GkyyOgq;-X}Te!YT{BDfh~XQifqT(6J>_B6;*5ZCA#rYZP3DR6{PIBo2a+&gNY2kKC`v6Z1_ga$i9$|lS!xc*edU=Y8Tq9p z$nj*Q5R_PwnQyDAXOyO(paF{wO;GH?%|E1NaTSW1&K5TJSL{6=A{-v zQiN(L!WEgR#X3+URiWXZpQezTnwwu#sldgckdz3rF)=+=!3xAxEmla&%t_5l%uQ8@ z)^$OcQyHtk#h~lLP?Q7@0dNQxGoXicUJ@i^ixq6ai5RR5Dx?ROWJpdfsQ~#MOUg?I zYXJqkCM21sx`4vpIX@>S6`TneQb1vXNG}j>G)T}dDHd)jXqJe9K?XW&2JXx-Fff2< z&^(kNHw)v%2v$Z0eg*~`1_%btNP}`Oh!3jmcog@=>RAl1*KDXdR=B1<-F@WY%8HywX z9pa;0Je@=0ONtUR^W2K^b0Nbd3=n6G{`Y9GdxgaU|<4|rzR(+#FrK)rl&GwFf)8m zWngAlV9dbG@WB{lCj$dRgWLn=2Bry23%DjQg9gs0GBPmmF)}bb5QL0#t%33dBp`f{ zCH#!wIeqX57HGvhKO+Of0!fIxB52tGBLl++AqZas%6}jL;afrZp!M9KfiBP*ZjgS^ zco~TA%K%mfVuD==T0PDP76Nm@E83Cy-Dv8UqVcz(@lT-fZ=vx&q47bZame<8rXi8} zpqU*;$Rc~N13|NF$b3tPVo{|x#GAL@BufPzGC7;A6!IOGCWG*2dBeTSVb4omNj>1t`#)D@q zP>V@aF>pzTDui4#LS+zzB8mVg12Z6tK#DJj2xbuo5l0k$2tKktSb-NGACFYjA=MkO z@-99NT+qd*K})#!G;q-dqEZ>+(;$UeJhT*xM=8GIi%Y=e7Dnv@4Qxm`6`xj=pX-*H Qlgbbe4?^%v3piB(0Dn~WwEzGB delta 3292 zcmcbVyDMwL1t9}b1_nk31_m{D1~4!Xo%qO3C`<+-ng$kQU26LS|`3k;!|Rt(k0CC;wu$atvT(U}!L8U`SwKU;r5>!N9=q z!jORhME@~nU~ph$VBlt8V1OA4HJgEzfdPz#L9SxWFlJy-ofs%Qc?FAxz>?dWjw}zf zS~6+dj;A(WQ`by>z@o;)ATybRwNL1QAp-*k0|SF50|SEq*s9G36>Rw@A7J&I{EyFu z*MXgZ!L#$IPv`R&rtFj5`9rxbu`w`!xTn}AFXR7cd7F)a!K2so0hr=_!N$O_s~2R> zLROHt?H;hW=|M2XdmSV_6(rrnI{AWtbiGI88x2MV29M6)KHa*PI2af_yK6zltz%&(Ru$x3M&Ic>jD1O7$yb={_U+-{{8>2%J6}K!Iq(cfq}o( zl?h}kT(ucgwH|iW;`K}n3_jgk4gUZC@7etCKYt4wSik8Jc95Iyzqrf-^8In&*8lu1 zuNgu9{{G@FSnP;z>yr|NeV}yZ)A`(|^OH~K&lek57#MskKa}u+!e=E51B0vKH&??a zKArzxG=ro%mx8?E)4BD=zyJSzI=4Qk|M&mDZ|f6~r} zt`j^uV;6XI#?J8Qc3t4n33i<81dndt8OY8``2WBDzfb447kpq>%}x0K|G!7;ZT=Pq zCU`n%W@TX51+v-jzzcu4D`Fw8@NNC((HXmgzhwpkC@6dzz@GH!{O;44dcddikw>TR z37>A)3okB%4D37tij_}Hp!8Ex?%D0j;L%%e8^p@MuuqzSfngUYDqs9#Vqn+}V!adq z#Rh-)fi!+iNVpyc2Q?_WK|@KN1(rs)!E|-r@Mt^&O3=}cF^;j0agOn^hr!1Abk{b# zJj%epVE7H>ifE97_}3q=2N{*-!Jl^k6z37lkT_2O(~vk%0R?Zb=?NB4oO^VKUU=cf z#K5qNnE{*_eLA1LV1b1D1+QM4zaZDWa0D6NdDOS{Eq{v`BLjnP>;DpC!`t9+dG_Kq zBPa%4?t&t%q`uDM|6w1?-$jpoI$wKqTfevf%8bp2IXwTLFMaRR`ToUAkU)3sg%=wb z85j(2`*f!s@ae8S;nVs3#W_%h=-!$Eini|B9WOwc0VKW+qGiL2wGhUN7fV1YJO96! z1E!z9n8L`w@bc2X|NrYfdToA2T)BFG!Pyjd&^47+xNN{Aqk?q1tpU~$tTFvSas zzFjv!(#8j6kw1R}i$EN*H7rg$Ym($7KCUqLCR*S3BxSi)2k zO!3YJ$^Qq*Ut?fk2=(ZE>KGObE}Km4LAk!0&!g9N5;FtCK0bJgxxfHQtacu~rX9?% z^stJ7fnf(I6-7J7#UAFDZvn*%L-T*8QcjQVI*yl2zyJRSn_eFl?9naf({1_?l%m1L zo1hwB1UFs_ZoChQ@&8r#Ff%axSKS9D4}i%-VDbo6P_mW?*m({;z7wJeglwsotM~f#Kze|NsAI zq=Q)dK&(~}YYT`q8^l@#Vr>Mm7Jyiw4DfOqh;x?{Qp108pL#){7||^LA6*$P@%xXz{DT|Za6S7Hn1@;C^9fHUW{O!{83)E{tUEj za}7#AfYPs^^cN`o4@z@Dn>9jES_Vq1L1_ahZ3U%WptK*9j)2lh5So=CN0xzM@<(~q z%@-8(KurumRzW3+1i{J72D$v8Fi&*`1rNgnh9ZN>=MBV}Ed(7VzcH0Cn7~lvAm|Vu z6&&IcpH`HLQ09Rw3JzP4=77nDhQjp`LJslq@rlL7sYNC6MJ0J4nFJw+__XAl{Nhv) zF9XEOFGvMT6@WMeMVWaeX&|`@5Eqn)K%53f2oJ2Q1I#He$}9nMCxE!Qi8(p>$snaO zK-}Vz#FBayPd|SbS2Gia2@DL3@eHgCEb+M+@u?N5$)zQ!@foR!DXB#ahTt^lRGOBS zTEt+Q#9)=1n3=~A28v~7h7DQ_3{2n@>ywxg4^CVRfy@jO^ca{KHkdOoGc;I0lE?Xnau7$p~rXf^E46kq3Dm%zuf-|A@-3 zXZV3Cz`)1|b~fAq4m3U=8Xr_UB0E41O&(OzAj^Z=*lY|844{q#2gBq96ERbMXuDaE zfq_AYfq_Aofq_AUfq_Akfq_Acfq?;3QHXl$i%gH4F?40Jqzgga7~l diff --git a/contrib/largeNbDicts/largeNbDicts.c b/contrib/largeNbDicts/largeNbDicts.c index 749d9660d..536e45ffe 100644 --- a/contrib/largeNbDicts/largeNbDicts.c +++ b/contrib/largeNbDicts/largeNbDicts.c @@ -159,6 +159,33 @@ buffer_collection_t splitBuffer(buffer_t srcBuffer, size_t blockSize) } +/*--- dictionary creation ---*/ + +buffer_t createDictionary(const char* dictionary, + const void* srcBuffer, size_t* srcBlockSizes, unsigned nbBlocks) +{ + if (dictionary) { + DISPLAYLEVEL(3, "loading dictionary %s \n", dictionary); + return createBuffer_fromFile(dictionary); + } else { + DISPLAYLEVEL(3, "creating dictionary, of target size %u bytes \n", DICTSIZE); + void* const dictBuffer = malloc(DICTSIZE); + assert(dictBuffer != NULL); + + size_t const dictSize = ZDICT_trainFromBuffer(dictBuffer, DICTSIZE, + srcBuffer, + srcBlockSizes, + nbBlocks); + assert(!ZSTD_isError(dictSize)); + + buffer_t result; + result.ptr = dictBuffer; + result.capacity = DICTSIZE; + result.size = dictSize; + return result; + } +} + /*--- ddict_collection_t ---*/ @@ -181,6 +208,7 @@ static void freeDDictCollection(ddict_collection_t ddictc) static ddict_collection_t createDDictCollection(const void* dictBuffer, size_t dictSize, size_t nbDDict) { ZSTD_DDict** const ddicts = malloc(nbDDict * sizeof(ZSTD_DDict*)); + assert(ddicts != NULL); if (ddicts==NULL) return kNullDDictCollection; for (size_t dictNb=0; dictNb < nbDDict; dictNb++) { ddicts[dictNb] = ZSTD_createDDict(dictBuffer, dictSize); @@ -193,29 +221,64 @@ static ddict_collection_t createDDictCollection(const void* dictBuffer, size_t d } +/* --- Compression --- */ -/*--- Benchmark --- */ +/* compressBlocks() : + * @return : total compressed size of all blocks, + * or 0 if error. + */ +static size_t compressBlocks(buffer_collection_t dstBlockBuffers, buffer_collection_t srcBlockBuffers, ZSTD_CDict* cdict, int cLevel) +{ + size_t const nbBlocks = srcBlockBuffers.nbBuffers; + assert(dstBlockBuffers.nbBuffers == srcBlockBuffers.nbBuffers); + + ZSTD_CCtx* const cctx = ZSTD_createCCtx(); + assert(cctx != NULL); + + size_t totalCSize = 0; + for (size_t blockNb=0; blockNb < nbBlocks; blockNb++) { + size_t cBlockSize; + if (cdict == NULL) { + cBlockSize = ZSTD_compressCCtx(cctx, + dstBlockBuffers.buffers[blockNb], dstBlockBuffers.capacities[blockNb], + srcBlockBuffers.buffers[blockNb], srcBlockBuffers.capacities[blockNb], + cLevel); + assert(!ZSTD_isError(cBlockSize)); + } else { + cBlockSize = ZSTD_compress_usingCDict(cctx, + dstBlockBuffers.buffers[blockNb], dstBlockBuffers.capacities[blockNb], + srcBlockBuffers.buffers[blockNb], srcBlockBuffers.capacities[blockNb], + cdict); + assert(!ZSTD_isError(cBlockSize)); + } + totalCSize += cBlockSize; + } + return totalCSize; +} + + +/* --- Benchmark --- */ /* bench() : + * fileName : file to load for benchmarking purpose + * dictionary : optional (can be NULL), file to load as dictionary, + * if none provided : will be calculated on the fly by the program. * @return : 0 is success, 1+ otherwise */ -int bench(const char* fileName) +int bench(const char* fileName, const char* dictionary) { int result = 0; DISPLAYLEVEL(3, "loading %s... \n", fileName); buffer_t const srcBuffer = createBuffer_fromFile(fileName); - if (srcBuffer.ptr == NULL) { - DISPLAYLEVEL(1," error reading file %s \n", fileName); - return 1; - } + assert(srcBuffer.ptr != NULL); DISPLAYLEVEL(3, "created src buffer of size %.1f MB \n", (double)(srcBuffer.size) / (1 MB)); buffer_collection_t const srcBlockBuffers = splitBuffer(srcBuffer, BLOCKSIZE); assert(srcBlockBuffers.buffers != NULL); unsigned const nbBlocks = (unsigned)srcBlockBuffers.nbBuffers; - DISPLAYLEVEL(3, "splitting input into %u blocks of max size %u bytes \n", + DISPLAYLEVEL(3, "split input into %u blocks of max size %u bytes \n", nbBlocks, BLOCKSIZE); size_t const dstBlockSize = ZSTD_compressBound(BLOCKSIZE); @@ -230,36 +293,44 @@ int bench(const char* fileName) buffer_collection_t const dstBlockBuffers = splitBuffer(dstBuffer, dstBlockSize); assert(dstBlockBuffers.buffers != NULL); - DISPLAYLEVEL(3, "creating dictionary, of target size %u bytes \n", DICTSIZE); - void* const dictBuffer = malloc(DICTSIZE); - if (dictBuffer == NULL) { result = 1; goto _cleanup; } + /* dictionary determination */ + buffer_t const dictBuffer = createDictionary(dictionary, + srcBuffer.ptr, + srcBlockBuffers.capacities, nbBlocks); + assert(dictBuffer.ptr != NULL); - size_t const dictSize = ZDICT_trainFromBuffer(dictBuffer, DICTSIZE, - srcBuffer.ptr, - srcBlockBuffers.capacities, - nbBlocks); - if (ZSTD_isError(dictSize)) { - DISPLAYLEVEL(1, "error creating dictionary \n"); - result = 1; - goto _cleanup; - } + ZSTD_CDict* const cdict = ZSTD_createCDict(dictBuffer.ptr, dictBuffer.size, COMP_LEVEL); + assert(cdict != NULL); - size_t const dictMem = ZSTD_estimateDDictSize(dictSize, ZSTD_dlm_byCopy); + size_t const cTotalSizeNoDict = compressBlocks(dstBlockBuffers, srcBlockBuffers, NULL, COMP_LEVEL); + assert(cTotalSizeNoDict != 0); + DISPLAYLEVEL(3, "compressing at level %u without dictionary : Ratio=%.2f (%u bytes) \n", + COMP_LEVEL, + (double)srcBuffer.size / cTotalSizeNoDict, (unsigned)cTotalSizeNoDict); + + size_t const cTotalSize = compressBlocks(dstBlockBuffers, srcBlockBuffers, cdict, COMP_LEVEL); + assert(cTotalSize != 0); + DISPLAYLEVEL(3, "compressed using a %u bytes dictionary : Ratio=%.2f (%u bytes) \n", + (unsigned)dictBuffer.size, + (double)srcBuffer.size / cTotalSize, (unsigned)cTotalSize); + + size_t const dictMem = ZSTD_estimateDDictSize(dictBuffer.size, ZSTD_dlm_byCopy); size_t const allDictMem = dictMem * nbBlocks; DISPLAYLEVEL(3, "generating %u dictionaries, using %.1f MB of memory \n", nbBlocks, (double)allDictMem / (1 MB)); - ZSTD_CDict* const cdict = ZSTD_createCDict(dictBuffer, dictSize, COMP_LEVEL); - do { - ddict_collection_t const dictionaries = createDDictCollection(dictBuffer, dictSize, nbBlocks); - assert(dictionaries.ddicts != NULL); + ddict_collection_t const dictionaries = createDDictCollection(dictBuffer.ptr, dictBuffer.size, nbBlocks); + assert(dictionaries.ddicts != NULL); - freeDDictCollection(dictionaries); - } while(0); + + + //result = benchMem(srcBlockBuffers, dstBlockBuffers, dictionaries);; + + + + freeDDictCollection(dictionaries); ZSTD_freeCDict(cdict); - -_cleanup: - free(dictBuffer); + freeBuffer(dictBuffer); freeCollection(dstBlockBuffers); freeBuffer(dstBuffer); freeCollection(srcBlockBuffers); @@ -276,7 +347,7 @@ _cleanup: int bad_usage(const char* exeName) { DISPLAY (" bad usage : \n"); - DISPLAY (" %s filename \n", exeName); + DISPLAY (" %s filename [-D dictionary] \n", exeName); return 1; } @@ -284,6 +355,15 @@ int main (int argc, const char** argv) { const char* const exeName = argv[0]; - if (argc != 2) return bad_usage(exeName); - return bench(argv[1]); + if (argc < 2) return bad_usage(exeName); + const char* const fileName = argv[1]; + + const char* dictionary = NULL; + if (argc > 2) { + if (argc != 4) return bad_usage(exeName); + if (strcmp(argv[2], "-D")) return bad_usage(exeName); + dictionary = argv[3]; + } + + return bench(fileName, dictionary); } From 0c66a44d1bcc0b7eae7f8ef52d6008541abdb7b1 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 28 Aug 2018 15:47:07 -0700 Subject: [PATCH 03/15] first working test program measures : - compression ratio with / without dictionary - create one dictionary per block - memory budget for dictionaries - decompression speed, using one different dictionary per block current limitations : - only one file - 4K blocks only - automatic dictionary built with 4K size dictionary can be selected on command line, with -D --- contrib/largeNbDicts/.gitignore | 2 + contrib/largeNbDicts/Makefile | 15 ++- contrib/largeNbDicts/largeNbDicts | Bin 14034 -> 0 bytes contrib/largeNbDicts/largeNbDicts.c | 156 ++++++++++++++++++++++++++-- programs/bench.c | 4 +- 5 files changed, 162 insertions(+), 15 deletions(-) create mode 100644 contrib/largeNbDicts/.gitignore delete mode 100755 contrib/largeNbDicts/largeNbDicts diff --git a/contrib/largeNbDicts/.gitignore b/contrib/largeNbDicts/.gitignore new file mode 100644 index 000000000..e77c4e496 --- /dev/null +++ b/contrib/largeNbDicts/.gitignore @@ -0,0 +1,2 @@ +# build artifacts +largeNbDicts diff --git a/contrib/largeNbDicts/Makefile b/contrib/largeNbDicts/Makefile index 026d76f12..f4b060ae0 100644 --- a/contrib/largeNbDicts/Makefile +++ b/contrib/largeNbDicts/Makefile @@ -7,8 +7,10 @@ # in the COPYING file in the root directory of this source tree). # ################################################################ +PROGDIR = ../../programs +LIBDIR = ../../lib -CPPFLAGS+= -I../../lib -I../../lib/common -I../../lib/dictBuilder -I../../programs +CPPFLAGS+= -I$(LIBDIR) -I$(LIBDIR)/common -I$(LIBDIR)/dictBuilder -I$(PROGDIR) CFLAGS ?= -O3 DEBUGFLAGS= -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \ @@ -24,9 +26,18 @@ default: largeNbDicts all : largeNbDicts largeNbDicts: LDFLAGS += -lzstd -largeNbDicts: largeNbDicts.c +largeNbDicts: bench.o datagen.o xxhash.o largeNbDicts.c $(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@ +bench.o : $(PROGDIR)/bench.c + $(CC) $(CPPFLAGS) $(CFLAGS) $^ -c + +datagen.o: $(PROGDIR)/datagen.c + $(CC) $(CPPFLAGS) $(CFLAGS) $^ -c + +xxhash.o : $(LIBDIR)/common/xxhash.c + $(CC) $(CPPFLAGS) $(CFLAGS) $^ -c clean: + $(RM) *.o $(RM) largeNbDicts diff --git a/contrib/largeNbDicts/largeNbDicts b/contrib/largeNbDicts/largeNbDicts deleted file mode 100755 index c057a2b78aa551a2de831b6e304f8747a6ea3d0f..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 14034 zcmX^A>+L^w1_nlE28ISE1_lNJ1_p)+tPBjT39lG3DNC=b(pEm9Ek>YyrMd?=TJ18N?^eIWDV zGg5O3Qj4&-k3||{-Xo|1AU-JEpiom1pLq#AoKE<%9XC?wcYHF)sn4 zodLv0Hv=jKragc@%1 zI6#U)Sb>27rWeEo#iuBU0mbq0c{%aLmAOgzIq?N0MGW!rsP5x{x(}3pKw3b2bo0bO z5>Ol;pOc8sJPD|IE1>E@d}Q-L{*{2rfhbV8LGr1Or=Pd0izh6P8K8xm0Z26i!xfMR z85kHq=791GM3jL+iGiU3ti*tU0TebI1`G_av;|UQV8FnzgOP#Zg&_k&1p@;EC@w+n zC@^ARkY`|E5HV$7IKarj0Lp#=APrD8Aax)sL1v0W#j#NdCJYR^SS0uu7{DbG4+8^( zera)$eokhReoAFd3RJB$0|Nud9d9k&nJ<5}jb4!?>6xKZt*!zs(-^oI7`Pa;U_7WU z4F-k=kggA);Dd^RD3C1``QgCH)Noua16I2VxS_Uqdf>B~L z1V%$(Gz3ONU^E0qLtr!nMniz25D4|?eCinP80Hx27!vH!{6@l~^Rq{1?FEl+R|$_! z*AqUyCCvW~FZpzS@c91Ov-7x5ugX!7g+86nUu@uJVDRib3Suq=GmrSTzAbV0ZGBSW zu$O zqXFd@fH($Fjs=Kg;lX&rqxlerNAs~CrTreQCrcDOdrhu*_J&+#nBdXL?0Cy8Qj&-C6VJ2go-*ofD16#^d3M0Hf}}rjGBBj2rRnj@ zw}68NC4K`uJI{M|Uh(Ms?AiIxqxH5&H|rlhP{>-8$a(adx^puy7#{HGy!B!qCn(&% zdvxA=v73Q`;dP8l=V6atQ#+6~p#G-effutm85s71*e_Ocf|CI#?%no*M0`5m`*c3? z>HO)~dEUnm?ad;kg6M!iH;L&`91J(OXV2_zL zf_=-&3bJ@D$bUj0<9cl$gT+mgz!dLKE(V5On?cf_Il$h>;{T9fkH$9>KUAFfj0MZw>kX|Gz532L=XP&^QTy>kN<{gzAG()s5Ixm*)9& zmu~QE{`bEm#iQ4>l?UYa`!6y%Kp}M8xAlLCuTSUq7v&rv@xvaiw@VcEf%;V*o%ek@ zKlyb2eBs5xz~E!~p@bLYGG`7323NyxhPQn>|G$s}8zQ6P+gYN*0U`umOL-o5Q2}QN z5CfbeJdd-0I(!TsjYmM91qCWNMPW@39*u8afHGp|H;>NWKHawOc^DWxyX`%CP4{!c zlEP+oP&01m8sFV_6~|KFp#6~c90;n8}4zhyNuSleg*y55Zc|Np1)C-F`N#RGo~ z?_4llbMh0v0P8|9TeBAw?w|N04|?>Pws0~qY+!uh53-}XwE%2~i;4iuf^M*R9-Tfa z93Gu4DjvOi75@MK4^Cq*oPUEI0uu1(bUgreh$Ay7G0!#l|NlS48=0I83_C$-%3sHL*)Oo;KwPlHo-u(U-1UM-_goE-alLyvz$(9a zbce3+=)CFE?YiMb;Gh5heY&@TWPDq{@wZ$6Yl=PL(^)&gr+cZu|NsAYfd)W)I=_SL zc)|)ca|hUh!%PeeKHYmAK!$fu^#D<@gwS1k0aWUChA!~v_C4U!?RvtaJ9dL-=gk)a zpy28}2r7147J@A6W_=6FP~8@39{hPHLGgH&je#MJKZf@zn65dQ#;?%|iN82d{4M2R zU;w*m1LKP>kbAmacYtFGY-#Nbkj32yi$Rvo23Z8Q)U)$Ge~Sl*+s#@EvbXcFN4GWH z>L9R{F}zV=8pG-a4p7o@>3sA;kd=YK@EfQWy34}A;K{%KDAbq(Y5Ym7++gi7tioWr z=D;WZNLFz+P?|QahZ^zg#W@xR2CrV5x1b{P#Z8bYoscwef{}p%RIqs(-cI9}cL5~` zaNy^H)v(+L2YwHzg!q5h$MSX21E0>{X~^ZTN4NEhAQlD&kLJT1p8wC5z6WLHY>+^A z?T!~lEMOB~L_q{DyaHe{93ou;kzNjw)&NQC zfTS%T(xDLP4v4e@NLn8xEeVk}gGgsWr1?S8Mj+{5pmfx0D-MzNf=K&-q(Nh;FK$Dm zQ$PhxucOqnUe+#H0GQ90-_}`4J7>%B<%%}PJ>9#f=JH;NxuO}Ycn!1fb$zTKEi^*^^>VT zsC4e;^XRqx#lpa_4>TgW3sil+07Zspx1C3?=?6%j=)Cpf7XxIRmLb|PF7_}iyujv# zJBEQ;yPcnXx{aIvGuEm4bXRk@8a`?M&r}lH{GX*%#G|`fz@yjnAPWP-i>nNv&?<58 z=$1VPa&9-bN3U%qssrYOihyowk6zP!umgNLKfS1AU|?|E19IPM0mE;vSv*KVu9NIFMB{NMn(pPmrWp+D2N5BO*1q>tUQpIIf#`4V!47?Q6N?*hy|*AGLk_o zH;`BXh-Cv})q_|@AXXoU1*&E<7K2!F|Ns97c?gCLoEaDlTwy#24H5&fVB#C>iVJcw zOPos6(o%~UauSQuQ~i=$GLuV+^^zI#lALn#le3Ez>}(Yb8S+xg!Q6n7A_W^;h2o-Q z*Sr*loczR;%)E4kl+5Ik%>2B>qDlqTVg)XSyEth5R&y5|9-o3dNaKsS2v4 z3Q3hEsc_|~Ad4Y}Fo4X1xJp6MR>3dS$A>{NDmcU?KC{@hs3^Zk1Ee50vno{+?gZ6h zJv}`IE{3GkyyOgq;-X}Te!YT{BDfh~XQifqT(6J>_B6;*5ZCA#rYZP3DR6{PIBo2a+&gNY2kKC`v6Z1_ga$i9$|lS!xc*edU=Y8Tq9p z$nj*Q5R_PwnQyDAXOyO(paF{wO;GH?%|E1NaTSW1&K5TJSL{6=A{-v zQiN(L!WEgR#X3+URiWXZpQezTnwwu#sldgckdz3rF)=+=!3xAxEmla&%t_5l%uQ8@ z)^$OcQyHtk#h~lLP?Q7@0dNQxGoXicUJ@i^ixq6ai5RR5Dx?ROWJpdfsQ~#MOUg?I zYXJqkCM21sx`4vpIX@>S6`TneQb1vXNG}j>G)T}dDHd)jXqJe9K?XW&2JXx-Fff2< z&^(kNHw)v%2v$Z0eg*~`1_%btNP}`Oh!3jmcog@=>RAl1*KDXdR=B1<-F@WY%8HywX z9pa;0Je@=0ONtUR^W2K^b0Nbd3=n6G{`Y9GdxgaU|<4|rzR(+#FrK)rl&GwFf)8m zWngAlV9dbG@WB{lCj$dRgWLn=2Bry23%DjQg9gs0GBPmmF)}bb5QL0#t%33dBp`f{ zCH#!wIeqX57HGvhKO+Of0!fIxB52tGBLl++AqZas%6}jL;afrZp!M9KfiBP*ZjgS^ zco~TA%K%mfVuD==T0PDP76Nm@E83Cy-Dv8UqVcz(@lT-fZ=vx&q47bZame<8rXi8} zpqU*;$Rc~N13|NF$b3tPVo{|x#GAL@BufPzGC7;A6!IOGCWG*2dBeTSVb4omNj>1t`#)D@q zP>V@aF>pzTDui4#LS+zzB8mVg12Z6tK#DJj2xbuo5l0k$2tKktSb-NGACFYjA=MkO z@-99NT+qd*K})#!G;q-dqEZ>+(;$UeJhT*xM=8GIi%Y=e7Dnv@4Qxm`6`xj=pX-*H Qlgbbe4?^%v3piB(0Dn~WwEzGB diff --git a/contrib/largeNbDicts/largeNbDicts.c b/contrib/largeNbDicts/largeNbDicts.c index 536e45ffe..168766018 100644 --- a/contrib/largeNbDicts/largeNbDicts.c +++ b/contrib/largeNbDicts/largeNbDicts.c @@ -12,7 +12,7 @@ * This is a benchmark test tool * dedicated to the specific case of dictionary decompression * using a very large nb of dictionaries - * thus generating many cache-misses. + * thus suffering latency from lots of cache misses. * It's created in a bid to investigate performance and find optimizations. */ @@ -24,6 +24,7 @@ #include /* assert */ #include "util.h" +#include "bench.h" #define ZSTD_STATIC_LINKING_ONLY #include "zstd.h" #include "zdict.h" @@ -158,6 +159,17 @@ buffer_collection_t splitBuffer(buffer_t srcBuffer, size_t blockSize) return result; } +/* shrinkSizes() : + * update sizes in buffer collection */ +void shrinkSizes(buffer_collection_t collection, + const size_t* sizes) /* presumed same size as collection */ +{ + size_t const nbBlocks = collection.nbBuffers; + for (size_t blockNb = 0; blockNb < nbBlocks; blockNb++) { + assert(sizes[blockNb] <= collection.capacities[blockNb]); + collection.capacities[blockNb] = sizes[blockNb]; + } +} /*--- dictionary creation ---*/ @@ -221,13 +233,30 @@ static ddict_collection_t createDDictCollection(const void* dictBuffer, size_t d } +/* mess with adresses, so that linear scanning dictionaries != linear address scanning */ +void shuffleDictionaries(ddict_collection_t dicts) +{ + size_t const nbDicts = dicts.nbDDict; + for (size_t r=0; rdctx, + dst, dstCapacity, + src, srcSize, + di->dictionaries.ddicts[di->blockNb]); + + di->blockNb = di->blockNb + 1; + if (di->blockNb >= di->nbBlocks) di->blockNb = 0; + + return result; +} + + +#define BENCH_TIME_DEFAULT_MS 6000 +#define RUN_TIME_DEFAULT_MS 1000 + +static int benchMem(buffer_collection_t dstBlocks, + buffer_collection_t srcBlocks, + ddict_collection_t dictionaries) +{ + assert(dstBlocks.nbBuffers == srcBlocks.nbBuffers); + assert(dstBlocks.nbBuffers == dictionaries.nbDDict); + + double bestSpeed = 0.; + + BMK_timedFnState_t* const benchState = + BMK_createTimedFnState(BENCH_TIME_DEFAULT_MS, RUN_TIME_DEFAULT_MS); + decompressInstructions di = createDecompressInstructions(dictionaries); + + for (;;) { + BMK_runOutcome_t const outcome = BMK_benchTimedFn(benchState, + decompress, &di, + NULL, NULL, + dstBlocks.nbBuffers, + (const void* const *)srcBlocks.buffers, srcBlocks.capacities, + dstBlocks.buffers, dstBlocks.capacities, + NULL); + + assert(BMK_isSuccessful_runOutcome(outcome)); + BMK_runTime_t const result = BMK_extract_runTime(outcome); + U64 const dTime_ns = result.nanoSecPerRun; + double const dTime_sec = (double)dTime_ns / 1000000000; + size_t const srcSize = result.sumOfReturn; + double const dSpeed_MBps = (double)srcSize / dTime_sec / (1 MB); + if (dSpeed_MBps > bestSpeed) bestSpeed = dSpeed_MBps; + DISPLAY("Decompression Speed : %.1f MB/s \r", bestSpeed); + if (BMK_isCompleted_TimedFn(benchState)) break; + } + DISPLAY("\n"); + + freeDecompressInstructions(di); + BMK_freeTimedFnState(benchState); + + return 0; /* success */ +} + /* bench() : * fileName : file to load for benchmarking purpose @@ -272,8 +387,9 @@ int bench(const char* fileName, const char* dictionary) DISPLAYLEVEL(3, "loading %s... \n", fileName); buffer_t const srcBuffer = createBuffer_fromFile(fileName); assert(srcBuffer.ptr != NULL); + size_t const srcSize = srcBuffer.size; DISPLAYLEVEL(3, "created src buffer of size %.1f MB \n", - (double)(srcBuffer.size) / (1 MB)); + (double)srcSize / (1 MB)); buffer_collection_t const srcBlockBuffers = splitBuffer(srcBuffer, BLOCKSIZE); assert(srcBlockBuffers.buffers != NULL); @@ -302,17 +418,20 @@ int bench(const char* fileName, const char* dictionary) ZSTD_CDict* const cdict = ZSTD_createCDict(dictBuffer.ptr, dictBuffer.size, COMP_LEVEL); assert(cdict != NULL); - size_t const cTotalSizeNoDict = compressBlocks(dstBlockBuffers, srcBlockBuffers, NULL, COMP_LEVEL); + size_t const cTotalSizeNoDict = compressBlocks(NULL, dstBlockBuffers, srcBlockBuffers, NULL, COMP_LEVEL); assert(cTotalSizeNoDict != 0); DISPLAYLEVEL(3, "compressing at level %u without dictionary : Ratio=%.2f (%u bytes) \n", COMP_LEVEL, - (double)srcBuffer.size / cTotalSizeNoDict, (unsigned)cTotalSizeNoDict); + (double)srcSize / cTotalSizeNoDict, (unsigned)cTotalSizeNoDict); - size_t const cTotalSize = compressBlocks(dstBlockBuffers, srcBlockBuffers, cdict, COMP_LEVEL); + size_t* const cSizes = malloc(nbBlocks * sizeof(size_t)); + assert(cSizes != NULL); + + size_t const cTotalSize = compressBlocks(cSizes, dstBlockBuffers, srcBlockBuffers, cdict, COMP_LEVEL); assert(cTotalSize != 0); DISPLAYLEVEL(3, "compressed using a %u bytes dictionary : Ratio=%.2f (%u bytes) \n", (unsigned)dictBuffer.size, - (double)srcBuffer.size / cTotalSize, (unsigned)cTotalSize); + (double)srcSize / cTotalSize, (unsigned)cTotalSize); size_t const dictMem = ZSTD_estimateDDictSize(dictBuffer.size, ZSTD_dlm_byCopy); size_t const allDictMem = dictMem * nbBlocks; @@ -322,13 +441,28 @@ int bench(const char* fileName, const char* dictionary) ddict_collection_t const dictionaries = createDDictCollection(dictBuffer.ptr, dictBuffer.size, nbBlocks); assert(dictionaries.ddicts != NULL); + shuffleDictionaries(dictionaries); + // for (size_t u = 0; u < dictionaries.nbDDict; u++) DISPLAY("dict address : %p \n", dictionaries.ddicts[u]); /* check dictionary addresses */ + void* const resultPtr = malloc(srcSize); + assert(resultPtr != NULL); + buffer_t resultBuffer; + resultBuffer.ptr = resultPtr; + resultBuffer.capacity = srcSize; + resultBuffer.size = srcSize; - //result = benchMem(srcBlockBuffers, dstBlockBuffers, dictionaries);; + buffer_collection_t const resultBlockBuffers = splitBuffer(resultBuffer, BLOCKSIZE); + assert(resultBlockBuffers.buffers != NULL); + shrinkSizes(dstBlockBuffers, cSizes); + result = benchMem(resultBlockBuffers, dstBlockBuffers, dictionaries); + /* free all heap objects in reverse order */ + freeCollection(resultBlockBuffers); + free(resultPtr); freeDDictCollection(dictionaries); + free(cSizes); ZSTD_freeCDict(cdict); freeBuffer(dictBuffer); freeCollection(dstBlockBuffers); @@ -342,7 +476,7 @@ int bench(const char* fileName, const char* dictionary) -/*--- Command Line ---*/ +/* --- Command Line --- */ int bad_usage(const char* exeName) { diff --git a/programs/bench.c b/programs/bench.c index b3a8222dd..5ff9afac5 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -253,7 +253,7 @@ static size_t local_defaultCompress( /* `addArgs` is the context */ static size_t local_defaultDecompress( const void* srcBuffer, size_t srcSize, - void* dstBuffer, size_t dstSize, + void* dstBuffer, size_t dstCapacity, void* addArgs) { size_t moreToFlush = 1; @@ -261,7 +261,7 @@ static size_t local_defaultDecompress( ZSTD_inBuffer in; ZSTD_outBuffer out; in.src = srcBuffer; in.size = srcSize; in.pos = 0; - out.dst = dstBuffer; out.size = dstSize; out.pos = 0; + out.dst = dstBuffer; out.size = dstCapacity; out.pos = 0; while (moreToFlush) { if(out.pos == out.size) { return (size_t)-ZSTD_error_dstSize_tooSmall; From 6c398df24118427bb802bb9b504a77e629c63831 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 28 Aug 2018 18:05:31 -0700 Subject: [PATCH 04/15] level, block size and nb dicts can be set on command line --- contrib/largeNbDicts/largeNbDicts.c | 111 +++++++++++++++++++++------- 1 file changed, 83 insertions(+), 28 deletions(-) diff --git a/contrib/largeNbDicts/largeNbDicts.c b/contrib/largeNbDicts/largeNbDicts.c index 168766018..4c05e24ec 100644 --- a/contrib/largeNbDicts/largeNbDicts.c +++ b/contrib/largeNbDicts/largeNbDicts.c @@ -35,9 +35,9 @@ #define KB *(1<<10) #define MB *(1<<20) -#define BLOCKSIZE (4 KB) +#define BLOCKSIZE_DEFAULT (4 KB) #define DICTSIZE (4 KB) -#define COMP_LEVEL 3 +#define CLEVEL_DEFAULT 3 #define DISPLAY_LEVEL_DEFAULT 3 @@ -293,8 +293,8 @@ typedef size_t (*BMK_initFn_t)(void* initPayload); typedef struct { ZSTD_DCtx* dctx; - size_t nbBlocks; - size_t blockNb; + size_t nbDicts; + size_t dictNb; ddict_collection_t dictionaries; } decompressInstructions; @@ -303,8 +303,8 @@ decompressInstructions createDecompressInstructions(ddict_collection_t dictionar decompressInstructions di; di.dctx = ZSTD_createDCtx(); assert(di.dctx != NULL); - di.nbBlocks = dictionaries.nbDDict; - di.blockNb = 0; + di.nbDicts = dictionaries.nbDDict; + di.dictNb = 0; di.dictionaries = dictionaries; return di; } @@ -322,10 +322,10 @@ size_t decompress(const void* src, size_t srcSize, void* dst, size_t dstCapacity size_t const result = ZSTD_decompress_usingDDict(di->dctx, dst, dstCapacity, src, srcSize, - di->dictionaries.ddicts[di->blockNb]); + di->dictionaries.ddicts[di->dictNb]); - di->blockNb = di->blockNb + 1; - if (di->blockNb >= di->nbBlocks) di->blockNb = 0; + di->dictNb = di->dictNb + 1; + if (di->dictNb >= di->nbDicts) di->dictNb = 0; return result; } @@ -339,7 +339,6 @@ static int benchMem(buffer_collection_t dstBlocks, ddict_collection_t dictionaries) { assert(dstBlocks.nbBuffers == srcBlocks.nbBuffers); - assert(dstBlocks.nbBuffers == dictionaries.nbDDict); double bestSpeed = 0.; @@ -364,6 +363,7 @@ static int benchMem(buffer_collection_t dstBlocks, double const dSpeed_MBps = (double)srcSize / dTime_sec / (1 MB); if (dSpeed_MBps > bestSpeed) bestSpeed = dSpeed_MBps; DISPLAY("Decompression Speed : %.1f MB/s \r", bestSpeed); + fflush(stdout); if (BMK_isCompleted_TimedFn(benchState)) break; } DISPLAY("\n"); @@ -380,7 +380,8 @@ static int benchMem(buffer_collection_t dstBlocks, * dictionary : optional (can be NULL), file to load as dictionary, * if none provided : will be calculated on the fly by the program. * @return : 0 is success, 1+ otherwise */ -int bench(const char* fileName, const char* dictionary) +int bench(const char* fileName, const char* dictionary, + size_t blockSize, int clevel, unsigned nbDictMax) { int result = 0; @@ -391,13 +392,13 @@ int bench(const char* fileName, const char* dictionary) DISPLAYLEVEL(3, "created src buffer of size %.1f MB \n", (double)srcSize / (1 MB)); - buffer_collection_t const srcBlockBuffers = splitBuffer(srcBuffer, BLOCKSIZE); + buffer_collection_t const srcBlockBuffers = splitBuffer(srcBuffer, blockSize); assert(srcBlockBuffers.buffers != NULL); unsigned const nbBlocks = (unsigned)srcBlockBuffers.nbBuffers; DISPLAYLEVEL(3, "split input into %u blocks of max size %u bytes \n", - nbBlocks, BLOCKSIZE); + nbBlocks, (unsigned)blockSize); - size_t const dstBlockSize = ZSTD_compressBound(BLOCKSIZE); + size_t const dstBlockSize = ZSTD_compressBound(blockSize); size_t const dstBufferCapacity = nbBlocks * dstBlockSize; void* const dstPtr = malloc(dstBufferCapacity); assert(dstPtr != NULL); @@ -415,30 +416,31 @@ int bench(const char* fileName, const char* dictionary) srcBlockBuffers.capacities, nbBlocks); assert(dictBuffer.ptr != NULL); - ZSTD_CDict* const cdict = ZSTD_createCDict(dictBuffer.ptr, dictBuffer.size, COMP_LEVEL); + ZSTD_CDict* const cdict = ZSTD_createCDict(dictBuffer.ptr, dictBuffer.size, clevel); assert(cdict != NULL); - size_t const cTotalSizeNoDict = compressBlocks(NULL, dstBlockBuffers, srcBlockBuffers, NULL, COMP_LEVEL); + size_t const cTotalSizeNoDict = compressBlocks(NULL, dstBlockBuffers, srcBlockBuffers, NULL, clevel); assert(cTotalSizeNoDict != 0); DISPLAYLEVEL(3, "compressing at level %u without dictionary : Ratio=%.2f (%u bytes) \n", - COMP_LEVEL, + clevel, (double)srcSize / cTotalSizeNoDict, (unsigned)cTotalSizeNoDict); size_t* const cSizes = malloc(nbBlocks * sizeof(size_t)); assert(cSizes != NULL); - size_t const cTotalSize = compressBlocks(cSizes, dstBlockBuffers, srcBlockBuffers, cdict, COMP_LEVEL); + size_t const cTotalSize = compressBlocks(cSizes, dstBlockBuffers, srcBlockBuffers, cdict, clevel); assert(cTotalSize != 0); DISPLAYLEVEL(3, "compressed using a %u bytes dictionary : Ratio=%.2f (%u bytes) \n", (unsigned)dictBuffer.size, (double)srcSize / cTotalSize, (unsigned)cTotalSize); size_t const dictMem = ZSTD_estimateDDictSize(dictBuffer.size, ZSTD_dlm_byCopy); - size_t const allDictMem = dictMem * nbBlocks; + unsigned const nbDicts = nbDictMax ? nbDictMax : nbBlocks; + size_t const allDictMem = dictMem * nbDicts; DISPLAYLEVEL(3, "generating %u dictionaries, using %.1f MB of memory \n", - nbBlocks, (double)allDictMem / (1 MB)); + nbDicts, (double)allDictMem / (1 MB)); - ddict_collection_t const dictionaries = createDDictCollection(dictBuffer.ptr, dictBuffer.size, nbBlocks); + ddict_collection_t const dictionaries = createDDictCollection(dictBuffer.ptr, dictBuffer.size, nbDicts); assert(dictionaries.ddicts != NULL); shuffleDictionaries(dictionaries); @@ -451,7 +453,7 @@ int bench(const char* fileName, const char* dictionary) resultBuffer.capacity = srcSize; resultBuffer.size = srcSize; - buffer_collection_t const resultBlockBuffers = splitBuffer(resultBuffer, BLOCKSIZE); + buffer_collection_t const resultBlockBuffers = splitBuffer(resultBuffer, blockSize); assert(resultBlockBuffers.buffers != NULL); shrinkSizes(dstBlockBuffers, cSizes); @@ -478,26 +480,79 @@ int bench(const char* fileName, const char* dictionary) /* --- Command Line --- */ +/*! readU32FromChar() : + * @return : unsigned integer value read from input in `char` format. + * allows and interprets K, KB, KiB, M, MB and MiB suffix. + * Will also modify `*stringPtr`, advancing it to position where it stopped reading. + * Note : function will exit() program if digit sequence overflows */ +static unsigned readU32FromChar(const char** stringPtr) +{ + unsigned result = 0; + while ((**stringPtr >='0') && (**stringPtr <='9')) { + unsigned const max = (((unsigned)(-1)) / 10) - 1; + assert(result <= max); /* check overflow */ + result *= 10, result += **stringPtr - '0', (*stringPtr)++ ; + } + if ((**stringPtr=='K') || (**stringPtr=='M')) { + unsigned const maxK = ((unsigned)(-1)) >> 10; + assert(result <= maxK); /* check overflow */ + result <<= 10; + if (**stringPtr=='M') { + assert(result <= maxK); /* check overflow */ + result <<= 10; + } + (*stringPtr)++; /* skip `K` or `M` */ + if (**stringPtr=='i') (*stringPtr)++; + if (**stringPtr=='B') (*stringPtr)++; + } + return result; +} + +/** longCommandWArg() : + * check if *stringPtr is the same as longCommand. + * If yes, @return 1 and advances *stringPtr to the position which immediately follows longCommand. + * @return 0 and doesn't modify *stringPtr otherwise. + */ +static unsigned longCommandWArg(const char** stringPtr, const char* longCommand) +{ + size_t const comSize = strlen(longCommand); + int const result = !strncmp(*stringPtr, longCommand, comSize); + if (result) *stringPtr += comSize; + return result; +} + + int bad_usage(const char* exeName) { DISPLAY (" bad usage : \n"); - DISPLAY (" %s filename [-D dictionary] \n", exeName); + DISPLAY (" %s filename [Options] \n", exeName); + DISPLAY ("Options : \n"); + DISPLAY ("--clevel=# : use compression level # (default: %u) \n", CLEVEL_DEFAULT); + DISPLAY ("--blockSize=# : cut input into blocks of size # (default: %u) \n", BLOCKSIZE_DEFAULT); + DISPLAY ("--dictionary=# : use # as a dictionary (default: create one) \n"); + DISPLAY ("--nbDicts=# : set nb of dictionaries to # (default: one per block) \n"); return 1; } int main (int argc, const char** argv) { const char* const exeName = argv[0]; + int cLevel = CLEVEL_DEFAULT; + size_t blockSize = BLOCKSIZE_DEFAULT; + size_t nbDicts = 0; if (argc < 2) return bad_usage(exeName); const char* const fileName = argv[1]; const char* dictionary = NULL; - if (argc > 2) { - if (argc != 4) return bad_usage(exeName); - if (strcmp(argv[2], "-D")) return bad_usage(exeName); - dictionary = argv[3]; + for (int argNb = 2; argNb < argc ; argNb++) { + const char* argument = argv[argNb]; + if (longCommandWArg(&argument, "--clevel=")) { cLevel = readU32FromChar(&argument); continue; } + if (longCommandWArg(&argument, "--blockSize=")) { blockSize = readU32FromChar(&argument); continue; } + if (longCommandWArg(&argument, "--dictionary=")) { dictionary = argument; continue; } + if (longCommandWArg(&argument, "--nbDicts=")) { nbDicts = readU32FromChar(&argument); continue; } + return bad_usage(exeName); } - return bench(fileName, dictionary); + return bench(fileName, dictionary, blockSize, cLevel, nbDicts); } From 6444c50035c466580ecbd90b5596288204a7d363 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 28 Aug 2018 18:13:46 -0700 Subject: [PATCH 05/15] increases randomness of ddict ptrs --- contrib/largeNbDicts/largeNbDicts.c | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/contrib/largeNbDicts/largeNbDicts.c b/contrib/largeNbDicts/largeNbDicts.c index 4c05e24ec..193362493 100644 --- a/contrib/largeNbDicts/largeNbDicts.c +++ b/contrib/largeNbDicts/largeNbDicts.c @@ -237,6 +237,12 @@ static ddict_collection_t createDDictCollection(const void* dictBuffer, size_t d void shuffleDictionaries(ddict_collection_t dicts) { size_t const nbDicts = dicts.nbDDict; + for (size_t r=0; r 0); + size_t* const fileSizes = (size_t*)calloc(nbFiles, sizeof(*fileSizes)); + assert(fileSizes != NULL); + + /* Load input buffer */ + int const errorCode = loadFiles(srcBuffer, loadedSize, + fileSizes, + fileNamesTable, nbFiles); + assert(errorCode == 0); + + void** sliceTable = (void**)malloc(nbFiles * sizeof(*sliceTable)); + assert(sliceTable != NULL); + + char* const ptr = (char*)srcBuffer; + size_t pos = 0; + unsigned fileNb = 0; + for ( ; (pos < loadedSize) && (fileNb < nbFiles); fileNb++) { + sliceTable[fileNb] = ptr + pos; + pos += fileSizes[fileNb]; + } + assert(pos == loadedSize); + assert(fileNb == nbFiles); + + + buffer_t buffer; + buffer.ptr = srcBuffer; + buffer.capacity = loadedSize; + buffer.size = loadedSize; + + slice_collection_t slices; + slices.slicePtrs = sliceTable; + slices.capacities = fileSizes; + slices.nbSlices = nbFiles; + + buffer_collection_t bc; + bc.buffer = buffer; + bc.slices = slices; + return bc; +} + + + + /*--- ddict_collection_t ---*/ typedef struct { @@ -260,12 +438,12 @@ void shuffleDictionaries(ddict_collection_t dicts) * or 0 if error. */ static size_t compressBlocks(size_t* cSizes, /* optional (can be NULL). If present, must contain at least nbBlocks fields */ - buffer_collection_t dstBlockBuffers, - buffer_collection_t srcBlockBuffers, + slice_collection_t dstBlockBuffers, + slice_collection_t srcBlockBuffers, ZSTD_CDict* cdict, int cLevel) { - size_t const nbBlocks = srcBlockBuffers.nbBuffers; - assert(dstBlockBuffers.nbBuffers == srcBlockBuffers.nbBuffers); + size_t const nbBlocks = srcBlockBuffers.nbSlices; + assert(dstBlockBuffers.nbSlices == srcBlockBuffers.nbSlices); ZSTD_CCtx* const cctx = ZSTD_createCCtx(); assert(cctx != NULL); @@ -275,16 +453,16 @@ static size_t compressBlocks(size_t* cSizes, /* optional (can be NULL). If pre size_t cBlockSize; if (cdict == NULL) { cBlockSize = ZSTD_compressCCtx(cctx, - dstBlockBuffers.buffers[blockNb], dstBlockBuffers.capacities[blockNb], - srcBlockBuffers.buffers[blockNb], srcBlockBuffers.capacities[blockNb], + dstBlockBuffers.slicePtrs[blockNb], dstBlockBuffers.capacities[blockNb], + srcBlockBuffers.slicePtrs[blockNb], srcBlockBuffers.capacities[blockNb], cLevel); } else { cBlockSize = ZSTD_compress_usingCDict(cctx, - dstBlockBuffers.buffers[blockNb], dstBlockBuffers.capacities[blockNb], - srcBlockBuffers.buffers[blockNb], srcBlockBuffers.capacities[blockNb], + dstBlockBuffers.slicePtrs[blockNb], dstBlockBuffers.capacities[blockNb], + srcBlockBuffers.slicePtrs[blockNb], srcBlockBuffers.capacities[blockNb], cdict); } - assert(!ZSTD_isError(cBlockSize)); + CONTROL(!ZSTD_isError(cBlockSize)); if (cSizes) cSizes[blockNb] = cBlockSize; totalCSize += cBlockSize; } @@ -337,31 +515,32 @@ size_t decompress(const void* src, size_t srcSize, void* dst, size_t dstCapacity } -#define BENCH_TIME_DEFAULT_MS 6000 -#define RUN_TIME_DEFAULT_MS 1000 - -static int benchMem(buffer_collection_t dstBlocks, - buffer_collection_t srcBlocks, - ddict_collection_t dictionaries) +static int benchMem(slice_collection_t dstBlocks, + slice_collection_t srcBlocks, + ddict_collection_t dictionaries, + int nbRounds) { - assert(dstBlocks.nbBuffers == srcBlocks.nbBuffers); + assert(dstBlocks.nbSlices == srcBlocks.nbSlices); + + unsigned const ms_per_round = RUN_TIME_DEFAULT_MS; + unsigned const total_time_ms = nbRounds * ms_per_round; double bestSpeed = 0.; BMK_timedFnState_t* const benchState = - BMK_createTimedFnState(BENCH_TIME_DEFAULT_MS, RUN_TIME_DEFAULT_MS); + BMK_createTimedFnState(total_time_ms, ms_per_round); decompressInstructions di = createDecompressInstructions(dictionaries); for (;;) { BMK_runOutcome_t const outcome = BMK_benchTimedFn(benchState, decompress, &di, NULL, NULL, - dstBlocks.nbBuffers, - (const void* const *)srcBlocks.buffers, srcBlocks.capacities, - dstBlocks.buffers, dstBlocks.capacities, + dstBlocks.nbSlices, + (const void* const *)srcBlocks.slicePtrs, srcBlocks.capacities, + dstBlocks.slicePtrs, dstBlocks.capacities, NULL); + CONTROL(BMK_isSuccessful_runOutcome(outcome)); - assert(BMK_isSuccessful_runOutcome(outcome)); BMK_runTime_t const result = BMK_extract_runTime(outcome); U64 const dTime_ns = result.nanoSecPerRun; double const dTime_sec = (double)dTime_ns / 1000000000; @@ -381,65 +560,87 @@ static int benchMem(buffer_collection_t dstBlocks, } -/* bench() : - * fileName : file to load for benchmarking purpose - * dictionary : optional (can be NULL), file to load as dictionary, +/*! bench() : + * fileName : file to load for benchmarking purpose + * dictionary : optional (can be NULL), file to load as dictionary, * if none provided : will be calculated on the fly by the program. * @return : 0 is success, 1+ otherwise */ -int bench(const char* fileName, const char* dictionary, - size_t blockSize, int clevel, unsigned nbDictMax) +int bench(const char** fileNameTable, unsigned nbFiles, + const char* dictionary, + size_t blockSize, int clevel, unsigned nbDictMax, int nbRounds) { int result = 0; - DISPLAYLEVEL(3, "loading %s... \n", fileName); - buffer_t const srcBuffer = createBuffer_fromFile(fileName); - assert(srcBuffer.ptr != NULL); + DISPLAYLEVEL(3, "loading %u files... \n", nbFiles); + buffer_collection_t const srcs = createBufferCollection_fromFiles(fileNameTable, nbFiles); + CONTROL(srcs.buffer.ptr != NULL); + buffer_t srcBuffer = srcs.buffer; size_t const srcSize = srcBuffer.size; DISPLAYLEVEL(3, "created src buffer of size %.1f MB \n", (double)srcSize / (1 MB)); - buffer_collection_t const srcBlockBuffers = splitBuffer(srcBuffer, blockSize); - assert(srcBlockBuffers.buffers != NULL); - unsigned const nbBlocks = (unsigned)srcBlockBuffers.nbBuffers; - DISPLAYLEVEL(3, "split input into %u blocks of max size %u bytes \n", - nbBlocks, (unsigned)blockSize); + slice_collection_t const srcSlices = splitSlices(srcs.slices, blockSize); + unsigned const nbBlocks = (unsigned)(srcSlices.nbSlices); + DISPLAYLEVEL(3, "split input into %u blocks ", nbBlocks); + if (blockSize) + DISPLAYLEVEL(3, "of max size %u bytes ", (unsigned)blockSize); + DISPLAYLEVEL(3, "\n"); - size_t const dstBlockSize = ZSTD_compressBound(blockSize); - size_t const dstBufferCapacity = nbBlocks * dstBlockSize; - void* const dstPtr = malloc(dstBufferCapacity); - assert(dstPtr != NULL); - buffer_t dstBuffer; - dstBuffer.ptr = dstPtr; - dstBuffer.capacity = dstBufferCapacity; - dstBuffer.size = dstBufferCapacity; - buffer_collection_t const dstBlockBuffers = splitBuffer(dstBuffer, dstBlockSize); - assert(dstBlockBuffers.buffers != NULL); + size_t* const dstCapacities = malloc(nbBlocks * sizeof(*dstCapacities)); + CONTROL(dstCapacities != NULL); + size_t dstBufferCapacity = 0; + for (size_t bnb=0; bnb bufferSize-pos) fileSize = bufferSize-pos, nbFiles=n; /* buffer too small - stop after this file */ - { size_t const readSize = fread(((char*)buffer)+pos, 1, (size_t)fileSize, f); - if (readSize != (size_t)fileSize) EXM_THROW_INT(11, "could not read %s", fileNamesTable[n]); - pos += readSize; } + { size_t const readSize = fread(((char*)buffer)+pos, 1, (size_t)fileSize, f); + if (readSize != (size_t)fileSize) EXM_THROW_INT(11, "could not read %s", fileNamesTable[n]); + pos += readSize; + } fileSizes[n] = (size_t)fileSize; totalSize += (size_t)fileSize; fclose(f); diff --git a/programs/util.h b/programs/util.h index 4392a5bd0..76000d991 100644 --- a/programs/util.h +++ b/programs/util.h @@ -526,7 +526,10 @@ UTIL_STATIC int UTIL_prepareFileList(const char *dirName, char** bufStart, size_ * After finishing usage of the list the structures should be freed with UTIL_freeFileList(params: return value, allocatedBuffer) * In case of error UTIL_createFileList returns NULL and UTIL_freeFileList should not be called. */ -UTIL_STATIC const char** UTIL_createFileList(const char **inputNames, unsigned inputNamesNb, char** allocatedBuffer, unsigned* allocatedNamesNb, int followLinks) +UTIL_STATIC const char** +UTIL_createFileList(const char **inputNames, unsigned inputNamesNb, + char** allocatedBuffer, unsigned* allocatedNamesNb, + int followLinks) { size_t pos; unsigned i, nbFiles; From 39ef91a599c4b68a724e37e55c49db930ff3305a Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Thu, 30 Aug 2018 14:59:10 -0700 Subject: [PATCH 09/15] -std=c99 for largeNbDicts --- Makefile | 1 + contrib/largeNbDicts/Makefile | 1 + 2 files changed, 2 insertions(+) diff --git a/Makefile b/Makefile index 45ab20fd8..03a26cd75 100644 --- a/Makefile +++ b/Makefile @@ -104,6 +104,7 @@ clean: @$(MAKE) -C contrib/pzstd $@ > $(VOID) @$(MAKE) -C contrib/seekable_format/examples $@ > $(VOID) @$(MAKE) -C contrib/adaptive-compression $@ > $(VOID) + @$(MAKE) -C contrib/largeNbDicts $@ > $(VOID) @$(RM) zstd$(EXT) zstdmt$(EXT) tmp* @$(RM) -r lz4 @echo Cleaning completed diff --git a/contrib/largeNbDicts/Makefile b/contrib/largeNbDicts/Makefile index cf5293991..0514e3d82 100644 --- a/contrib/largeNbDicts/Makefile +++ b/contrib/largeNbDicts/Makefile @@ -15,6 +15,7 @@ LIBZSTD = $(LIBDIR)/libzstd.a CPPFLAGS+= -I$(LIBDIR) -I$(LIBDIR)/common -I$(LIBDIR)/dictBuilder -I$(PROGDIR) CFLAGS ?= -O3 +CFLAGS += -std=c99 DEBUGFLAGS= -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \ -Wstrict-aliasing=1 -Wswitch-enum \ -Wstrict-prototypes -Wundef -Wpointer-arith -Wformat-security \ From 39c55a118f778c6120cdfca40689a7411ce9e6a4 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Thu, 30 Aug 2018 15:54:14 -0700 Subject: [PATCH 10/15] fixed minor compatibility issues with older compilers --- contrib/largeNbDicts/Makefile | 2 +- contrib/largeNbDicts/largeNbDicts.c | 5 ++--- programs/util.h | 10 ++++++---- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/contrib/largeNbDicts/Makefile b/contrib/largeNbDicts/Makefile index 0514e3d82..05b54fd35 100644 --- a/contrib/largeNbDicts/Makefile +++ b/contrib/largeNbDicts/Makefile @@ -15,7 +15,7 @@ LIBZSTD = $(LIBDIR)/libzstd.a CPPFLAGS+= -I$(LIBDIR) -I$(LIBDIR)/common -I$(LIBDIR)/dictBuilder -I$(PROGDIR) CFLAGS ?= -O3 -CFLAGS += -std=c99 +CFLAGS += -std=gnu99 DEBUGFLAGS= -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \ -Wstrict-aliasing=1 -Wswitch-enum \ -Wstrict-prototypes -Wundef -Wpointer-arith -Wformat-security \ diff --git a/contrib/largeNbDicts/largeNbDicts.c b/contrib/largeNbDicts/largeNbDicts.c index 5b982bd42..0c5f89a8e 100644 --- a/contrib/largeNbDicts/largeNbDicts.c +++ b/contrib/largeNbDicts/largeNbDicts.c @@ -50,6 +50,8 @@ /*--- Macros ---*/ #define CONTROL(c) assert(c) +#undef MIN +#define MIN(a,b) ((a) < (b) ? (a) : (b)) /*--- Display Macros ---*/ @@ -472,9 +474,6 @@ static size_t compressBlocks(size_t* cSizes, /* optional (can be NULL). If pre /* --- Benchmark --- */ -typedef size_t (*BMK_benchFn_t)(const void* src, size_t srcSize, void* dst, size_t dstCapacity, void* customPayload); -typedef size_t (*BMK_initFn_t)(void* initPayload); - typedef struct { ZSTD_DCtx* dctx; size_t nbDicts; diff --git a/programs/util.h b/programs/util.h index 76000d991..88d048409 100644 --- a/programs/util.h +++ b/programs/util.h @@ -319,15 +319,17 @@ UTIL_STATIC U32 UTIL_isDirectory(const char* infilename) UTIL_STATIC U32 UTIL_isLink(const char* infilename) { -#if defined(_WIN32) - /* no symlinks on windows */ - (void)infilename; -#else +/* macro guards, as defined in : https://linux.die.net/man/2/lstat */ +#if defined(_BSD_SOURCE) \ + || (defined(_XOPEN_SOURCE) && (_XOPEN_SOURCE >= 500)) \ + || (defined(_XOPEN_SOURCE) && defined(_XOPEN_SOURCE_EXTENDED)) \ + || (defined(_POSIX_C_SOURCE) && (_POSIX_C_SOURCE >= 200112L)) int r; stat_t statbuf; r = lstat(infilename, &statbuf); if (!r && S_ISLNK(statbuf.st_mode)) return 1; #endif + (void)infilename; return 0; } From f76253bb702bdaa6227e9285111f8c08fb4e9830 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Thu, 30 Aug 2018 16:24:44 -0700 Subject: [PATCH 11/15] minor : createDictionaryBuffer() can create dictionaries of different sizes --- contrib/largeNbDicts/largeNbDicts.c | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/contrib/largeNbDicts/largeNbDicts.c b/contrib/largeNbDicts/largeNbDicts.c index 0c5f89a8e..70013cdd8 100644 --- a/contrib/largeNbDicts/largeNbDicts.c +++ b/contrib/largeNbDicts/largeNbDicts.c @@ -131,25 +131,28 @@ static buffer_t createBuffer_fromFile(const char* fileName) static buffer_t createDictionaryBuffer(const char* dictionaryName, const void* srcBuffer, - const size_t* srcBlockSizes, unsigned nbBlocks) + const size_t* srcBlockSizes, unsigned nbBlocks, + size_t requestedDictSize) { if (dictionaryName) { DISPLAYLEVEL(3, "loading dictionary %s \n", dictionaryName); return createBuffer_fromFile(dictionaryName); - } else { - DISPLAYLEVEL(3, "creating dictionary, of target size %u bytes \n", DICTSIZE); - void* const dictBuffer = malloc(DICTSIZE); - assert(dictBuffer != NULL); - size_t const dictSize = ZDICT_trainFromBuffer(dictBuffer, DICTSIZE, - srcBuffer, - srcBlockSizes, - nbBlocks); - assert(!ZSTD_isError(dictSize)); + } else { + + DISPLAYLEVEL(3, "creating dictionary, of target size %u bytes \n", + (unsigned)requestedDictSize); + void* const dictBuffer = malloc(requestedDictSize); + CONTROL(dictBuffer != NULL); + + size_t const dictSize = ZDICT_trainFromBuffer(dictBuffer, requestedDictSize, + srcBuffer, + srcBlockSizes, nbBlocks); + CONTROL(!ZSTD_isError(dictSize)); buffer_t result; result.ptr = dictBuffer; - result.capacity = DICTSIZE; + result.capacity = requestedDictSize; result.size = dictSize; return result; } @@ -616,7 +619,8 @@ int bench(const char** fileNameTable, unsigned nbFiles, /* dictionary determination */ buffer_t const dictBuffer = createDictionaryBuffer(dictionary, srcBuffer.ptr, - srcSlices.capacities, nbBlocks); + srcSlices.capacities, nbBlocks, + DICTSIZE); CONTROL(dictBuffer.ptr != NULL); ZSTD_CDict* const cdict = ZSTD_createCDict(dictBuffer.ptr, dictBuffer.size, clevel); From 0ff67511e64fb81880544ffaa4d03af7125e3de2 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Thu, 30 Aug 2018 16:43:28 -0700 Subject: [PATCH 12/15] fixed link order for old compilers --- contrib/largeNbDicts/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/largeNbDicts/Makefile b/contrib/largeNbDicts/Makefile index 05b54fd35..624140fab 100644 --- a/contrib/largeNbDicts/Makefile +++ b/contrib/largeNbDicts/Makefile @@ -28,7 +28,7 @@ default: largeNbDicts all : largeNbDicts -largeNbDicts: bench.o datagen.o xxhash.o $(LIBZSTD) largeNbDicts.c +largeNbDicts: bench.o datagen.o xxhash.o largeNbDicts.c $(LIBZSTD) $(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@ .PHONY: $(LIBZSTD) From 11b8b8c10016902e216878223db8951c3c65441a Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Fri, 31 Aug 2018 10:01:06 -0700 Subject: [PATCH 13/15] silenced false-positive scan-build warning --- contrib/largeNbDicts/largeNbDicts.c | 39 +++++++++++++++-------------- 1 file changed, 20 insertions(+), 19 deletions(-) diff --git a/contrib/largeNbDicts/largeNbDicts.c b/contrib/largeNbDicts/largeNbDicts.c index 70013cdd8..943a644d6 100644 --- a/contrib/largeNbDicts/largeNbDicts.c +++ b/contrib/largeNbDicts/largeNbDicts.c @@ -74,6 +74,7 @@ static const buffer_t kBuffNull = { NULL, 0, 0 }; /* @return : kBuffNull if any error */ static buffer_t createBuffer(size_t capacity) { + assert(capacity > 0); void* const ptr = malloc(capacity); if (ptr==NULL) return kBuffNull; @@ -96,18 +97,6 @@ static void fillBuffer_fromHandle(buffer_t* buff, FILE* f) buff->size = readSize; } -/* @return : kBuffNull if any error */ -static buffer_t createBuffer_fromHandle(FILE* f, size_t bufferSize) -{ - buffer_t buff = createBuffer(bufferSize); - if (buff.ptr == NULL) return kBuffNull; - fillBuffer_fromHandle(&buff, f); - if (buff.size != buff.capacity) { - freeBuffer(buff); - return kBuffNull; - } - return buff; -} /* @return : kBuffNull if any error */ static buffer_t createBuffer_fromFile(const char* fileName) @@ -118,16 +107,22 @@ static buffer_t createBuffer_fromFile(const char* fileName) if (fileSize == UTIL_FILESIZE_UNKNOWN) return kBuffNull; assert((U64)bufferSize == fileSize); /* check overflow */ - { buffer_t buff; - FILE* const f = fopen(fileName, "rb"); + { FILE* const f = fopen(fileName, "rb"); if (f == NULL) return kBuffNull; - buff = createBuffer_fromHandle(f, bufferSize); + buffer_t buff = createBuffer(bufferSize); + CONTROL(buff.ptr != NULL); + + fillBuffer_fromHandle(&buff, f); + CONTROL(buff.size == buff.capacity); + fclose(f); /* do nothing specific if fclose() fails */ return buff; } } + +/* @return : kBuffNull if any error */ static buffer_t createDictionaryBuffer(const char* dictionaryName, const void* srcBuffer, @@ -136,7 +131,7 @@ createDictionaryBuffer(const char* dictionaryName, { if (dictionaryName) { DISPLAYLEVEL(3, "loading dictionary %s \n", dictionaryName); - return createBuffer_fromFile(dictionaryName); + return createBuffer_fromFile(dictionaryName); /* note : result might be kBuffNull */ } else { @@ -172,11 +167,11 @@ static int loadFiles(void* buffer, size_t bufferSize, for (unsigned n=0; n 0); void* const srcBuffer = malloc(loadedSize); assert(srcBuffer != NULL); @@ -777,5 +773,10 @@ int main (int argc, const char** argv) filenameTable = UTIL_createFileList(nameTable, nameIdx, &buffer_containing_filenames, &nbFiles, 1 /* follow_links */); } - return bench(filenameTable, nbFiles, dictionary, blockSize, cLevel, nbDicts, nbRounds); + int result = bench(filenameTable, nbFiles, dictionary, blockSize, cLevel, nbDicts, nbRounds); + + free(buffer_containing_filenames); + free(nameTable); + + return result; } From 1d487d587f3a8964e274ecd5062150c24deea98e Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 4 Sep 2018 14:57:45 -0700 Subject: [PATCH 14/15] updated documentation --- contrib/largeNbDicts/README.md | 22 ++++++++++++++-------- contrib/largeNbDicts/largeNbDicts.c | 26 ++++++++++++++++++-------- 2 files changed, 32 insertions(+), 16 deletions(-) diff --git a/contrib/largeNbDicts/README.md b/contrib/largeNbDicts/README.md index 7eba229a9..f29bcdfe8 100644 --- a/contrib/largeNbDicts/README.md +++ b/contrib/largeNbDicts/README.md @@ -3,17 +3,23 @@ largeNbDicts `largeNbDicts` is a benchmark test tool dedicated to the specific scenario of -dictionary decompression using a very large number of dictionaries, -which suffers from increased latency due to cache misses. -It's created in a bid to investigate performance for this scenario, +dictionary decompression using a very large number of dictionaries. +When dictionaries are constantly changing, they are always "cold", +suffering from increased latency due to cache misses. + +The tool is created in a bid to investigate performance for this scenario, and experiment mitigation techniques. Command line : ``` -$ largeNbDicts filename [Options] +largeNbDicts [Options] filename(s) + Options : ---clevel=# : use compression level # (default: 3) ---blockSize=# : cut input into blocks of size # (default: 4096) ---dictionary=# : use # as a dictionary (default: create one) ---nbDicts=# : set nb of dictionaries to # (default: one per block) +-r : recursively load all files in subdirectories (default: off) +-B# : split input into blocks of size # (default: no split) +-# : use compression level # (default: 3) +-D # : use # as a dictionary (default: create one) +-i# : nb benchmark rounds (default: 6) +--nbDicts=# : set nb of dictionaries to # (default: one per block) +-h : help (this text) ``` diff --git a/contrib/largeNbDicts/largeNbDicts.c b/contrib/largeNbDicts/largeNbDicts.c index 943a644d6..21fbd2aa5 100644 --- a/contrib/largeNbDicts/largeNbDicts.c +++ b/contrib/largeNbDicts/largeNbDicts.c @@ -716,17 +716,26 @@ static unsigned longCommandWArg(const char** stringPtr, const char* longCommand) } +int usage(const char* exeName) +{ + DISPLAY (" \n"); + DISPLAY (" %s [Options] filename(s) \n", exeName); + DISPLAY (" \n"); + DISPLAY ("Options : \n"); + DISPLAY ("-r : recursively load all files in subdirectories (default: off) \n"); + DISPLAY ("-B# : split input into blocks of size # (default: no split) \n"); + DISPLAY ("-# : use compression level # (default: %u) \n", CLEVEL_DEFAULT); + DISPLAY ("-D # : use # as a dictionary (default: create one) \n"); + DISPLAY ("-i# : nb benchmark rounds (default: %u) \n", BENCH_TIME_DEFAULT_S); + DISPLAY ("--nbDicts=# : create # dictionaries for bench (default: one per block) \n"); + DISPLAY ("-h : help (this text) \n"); + return 0; +} + int bad_usage(const char* exeName) { DISPLAY (" bad usage : \n"); - DISPLAY (" %s filename [Options] \n", exeName); - DISPLAY ("Options : \n"); - DISPLAY ("-r : recursively load all files in subdirectories (default: off) \n"); - DISPLAY ("-B# : split input into blocks of size # (default: no split) \n"); - DISPLAY ("-# : use compression level # (default: %u) \n", CLEVEL_DEFAULT); - DISPLAY ("-D # : use # as a dictionary (default: create one) \n"); - DISPLAY ("-i# : nb benchmark rounds (default: %u) \n", BENCH_TIME_DEFAULT_S); - DISPLAY ("--nbDicts=# : create # dictionaries for bench (default: one per block) \n"); + usage(exeName); return 1; } @@ -749,6 +758,7 @@ int main (int argc, const char** argv) for (int argNb = 1; argNb < argc ; argNb++) { const char* argument = argv[argNb]; + if (!strcmp(argument, "-h")) { return usage(exeName); } if (!strcmp(argument, "-r")) { recursiveMode = 1; continue; } if (!strcmp(argument, "-D")) { argNb++; assert(argNb < argc); dictionary = argv[argNb]; continue; } if (longCommandWArg(&argument, "-i")) { nbRounds = readU32FromChar(&argument); continue; } From c57a856d64174a199852481b73bbe20820bd3590 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 5 Sep 2018 14:33:51 -0700 Subject: [PATCH 15/15] fixed minor static analyzer warning --- contrib/largeNbDicts/largeNbDicts.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/largeNbDicts/largeNbDicts.c b/contrib/largeNbDicts/largeNbDicts.c index 21fbd2aa5..d094065a4 100644 --- a/contrib/largeNbDicts/largeNbDicts.c +++ b/contrib/largeNbDicts/largeNbDicts.c @@ -758,7 +758,7 @@ int main (int argc, const char** argv) for (int argNb = 1; argNb < argc ; argNb++) { const char* argument = argv[argNb]; - if (!strcmp(argument, "-h")) { return usage(exeName); } + if (!strcmp(argument, "-h")) { free(nameTable); return usage(exeName); } if (!strcmp(argument, "-r")) { recursiveMode = 1; continue; } if (!strcmp(argument, "-D")) { argNb++; assert(argNb < argc); dictionary = argv[argNb]; continue; } if (longCommandWArg(&argument, "-i")) { nbRounds = readU32FromChar(&argument); continue; }