From 678272515515afc6fe99975a8c716ce27907fc7e Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Sun, 26 Aug 2018 19:29:12 -0700 Subject: [PATCH] first sketch for largeNbDicts test program --- contrib/largeNbDicts/Makefile | 30 +++ contrib/largeNbDicts/largeNbDicts | Bin 0 -> 13626 bytes contrib/largeNbDicts/largeNbDicts.c | 289 ++++++++++++++++++++++++++++ lib/Makefile | 2 +- lib/dictBuilder/zdict.h | 3 +- programs/bench.h | 2 +- 6 files changed, 323 insertions(+), 3 deletions(-) create mode 100644 contrib/largeNbDicts/Makefile create mode 100755 contrib/largeNbDicts/largeNbDicts create mode 100644 contrib/largeNbDicts/largeNbDicts.c diff --git a/contrib/largeNbDicts/Makefile b/contrib/largeNbDicts/Makefile new file mode 100644 index 000000000..082f0102a --- /dev/null +++ b/contrib/largeNbDicts/Makefile @@ -0,0 +1,30 @@ +# ################################################################ +# Copyright (c) 2018-present, Yann Collet, Facebook, Inc. +# All rights reserved. +# +# This source code is licensed under both the BSD-style license (found in the +# LICENSE file in the root directory of this source tree) and the GPLv2 (found +# in the COPYING file in the root directory of this source tree). +# ################################################################ + + +CPPFLAGS+= -I../../lib -I../../lib/common -I../../lib/dictBuilder -I../../programs + +CFLAGS ?= -O3 +DEBUGFLAGS= -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \ + -Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \ + -Wstrict-prototypes -Wundef -Wpointer-arith -Wformat-security \ + -Wvla -Wformat=2 -Winit-self -Wfloat-equal -Wwrite-strings \ + -Wredundant-decls +CFLAGS += $(DEBUGFLAGS) $(MOREFLAGS) + + +default: largeNbDicts + +largeNbDicts: LDFLAGS += -lzstd +largeNbDicts: largeNbDicts.c + $(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@ + + +clean: + $(RM) largeNbDicts diff --git a/contrib/largeNbDicts/largeNbDicts b/contrib/largeNbDicts/largeNbDicts new file mode 100755 index 0000000000000000000000000000000000000000..40416f050fa594cb085444ebb1ca433629d59318 GIT binary patch literal 13626 zcmX^A>+L^w1_nlE28ISE1_lNJ1_p)+tPBjT3yFff4lEC}Tc3@i){$lUn&;*!#&Vz^LzJgRwNG7$69pekT|D3^f)Y97LU zAoJogQgaGYi?FzF57fL}Py;}GP`E?6SlpMKpI40VFuMC1WFhX;fEobeqxcsp4x{4Z zlZ#7=GV{`*0_f(gfSSh)brgsXRSKq{!eBOth>tJLE6>bJiO1BUlJRC_u~sCs2?GG=0GMpy+01U|;}YkaT=}dOjl5 zaGS>gQVhZh2)!UKC_V*23@DC|&&!D~uFOr!&xtQ6DPo9^M|B?u)P11*1JVNG!`uSm zgUk~GF`zg;J|_{Mc@j|bI-u%7d}Q-L{uP1BfhbV8LGr1Or=Pd0izh6P8K8xm0Z26i zgAFKCF)%QI%mL*Wh$sVt5(7g6Scw4x11M}b3>X+-X$z#rz<_}vfRTZr!H|I=fq{Vm z6qgbV3=A&}85ltHA7cgv2Sx@4ZUzPhSe%3OfiTEQ5Ece;aNrDM1_o6ua(oO7;1Y?4 zfq_B4w75t=Co@Sur7|Z4s#cl-V*ircn~p3Gv|2K0+m5F;UQ^eoK+7}+E(QiJ1}zv5 zs!M}`p#kJ-1{tUTlmgk}05uWhLl$OG7%@U^&QDIv(a$d^(XT2lNdY?~Co>7e)-%*g zsm#etVgTv4K-O>cgas52aF^#~CIwd(m!#(EIYG66ZB~Q2?|~sC^fjS;Q0Rgcj1r?E zFd71*Aut*OqaiRF0;3@?8UmvsFd71*Aut*OqaiRF0yGbSP>;^1j^U1Bj-ier!5+2X+Pq&(5PD zrYV?t#JBZriMwy>lM*LS%X|Dyri=^>9=)yt9^DKc-L)q?7?1mO{`2^Mj=$wK0|SF+ zZ}`8?kIfI+J(_>97dseU@@PHD-@1{3f#KyD1_lP7&i5}av4QC4FHW&BFnDykUhp{X z03MF=IPL%%%3<(04sr{_%e|m6UCncbmnL|2x?E-O?3B6U!FU72=J4!{xxxYE2!J>O zP>uwMBLU?ofH(?Jjs}RM0p%EgI0jIT1&Cwe!Fa=?`4ERk^RXYL{T{6+OB6hNO|E$M zhFoQs;L*wC(J2R3af}13>i@~oS00u>%Dp@^k9&8z{QcqGS@Y)y$TvQn8a|x~96p^D z0v^prB%&Q-9Ah2h9OGjTd-U4g26?&H^Z}USeZj`Su&Wm|Dz%Um6sER&z~ZI{!4&Ux zkn~iLbQ3EBLt0vz9>07GIA~DgxABbzC`vki`*iDG;$UF#?5<_-=rvu(#=u~Bz@zil zi%_uXTQB_k|KFqY{)-gQxcLG8))*!R2LA1>SN{F~ugdU&fx#9u!?T(ucg zwH|iW;!F$-KHXal{{R2)+5GQ6e+wH}yXg^j1_qF2msvo`>9}v}fBu%&jG&1B{^BlJ z>3O-?#M%$jkgK%NQ9L(meU|j-~M@v3h{Qj6a4o08G~$_{1N{ z8UhXo)4i+=3>z3<*s*}pUn|H6k4_gA0gvuhkl`NPy&%IqI(<|)JUUraJbL$5{Qv)d zA7}{o#iBp||9f<|f&@G|T~~Ot9^h}81oClrXoq9xb&xwedQF$JGBE4}sRxCB=!F+o zm>C%MgZMA{m_fmD%%}6SC;$5Upa8nS%)kIPCJSUtr|SlfPS*(@ov{l%I%8*ebh|Eq zy3KWhN4M_`WLG8p2L-{m7kpqh%>{8iT5t2WI55G}Kr<@?!!D4;h6i5w!yOR|afEN{ zH;>NP9sDgbKpCyuw*l-$pU&?-ov8meY5Xy) z31Au$-zlK5?KM5Y0*Y^s?$8S_yqFjmc7X<{!QOoKf&~)j7rc6H{(>C$!VzS4=TYC* zxBM+)j0_CEt^Z4m4R1r0-)00wpUYiPjFr@R{6Fkt`Mc<`Pv>ioZtE8pK)JB_Fo);= z^QG@W1>H-KKzHqh7aJHE7z}Uwbf+Hh>8?HD)A{|yIZ#&U-kR|rl$>gJyZ~hbkoY!; zmJKh~LKrJvECH$P{QqJOn123Z3L^u<%S)iF+-sWzO2NIR6G5Kn<;`MYVA!=AR9*yu zH22#60*jj#fhk^4)a|+fk~W4&-v*1D27@VHO=bp$T@OIgA`t04U~y9uFvTkgl70@7 z{t8Mhy|#0~;-;ctigz|h`aeke8Uq6ZxO@r=_5c-0ruLxR-p%LHYdeXVfngtLgdCDu zE`X|wZaa@&(+*}>T3E%vz_0_9f}$PcVh_X0yXOB)rJNq!bsR65e*gauGre2Rr`z-) zC@q6cH$gQWG^^zaHeCyDx(|x!|5f)eGcf#D-3KNQfXPE(@(7qb1}0B{$x~qR446Cz zCPDf3zv?A08VIgJbZ2Ra<5Th6x#533dGU@u>wFWYY@xr|9?8XB6E}6+C#d^sMc}Y$=`N`SE z3U;;%h75VBu8T1< zpg>nGRse-`USe*l0vAJ35;U}3Kr!i@pOcdc_5uUMlxUDyeo3+L@C46l!=|Z_Y0&%> zXqMZAfq?;p!=aKOHmE@6VPIkq0ngJhFgCC;FeoxGFNf+7C)cKF9jr{0ha*}Fgb&UJV4?GsN&$E5RkY9Ogy!?Br`X$Bo!q@o z;0yqd{_rz`Cj-IbNMZ~O4E&4?3>&0DtCB%uLJ+=yIE1eOl?Sbh1`QPHLHVF@CJ^76 z0jv(h1SJcGAT&N`w*(_t30Rt;0!_XPjlT$uzX^?h2#pUK;bw#^J_lQH4A3Sa12Z*YEOU|;~Pk_WBu7iC~z0L2$*d|w>wI|c`8n5V!Hk0ss3C&5!xJY=9Z z9-Q*xk<%lNA|O5rGxI~mkg`3p2qc?AL@={DL>!UH5qxBQu#6oaACHu!k;*ezzK%}= k=jiw}XkLy_1LtB8mC6vG2FbVa&pF literal 0 HcmV?d00001 diff --git a/contrib/largeNbDicts/largeNbDicts.c b/contrib/largeNbDicts/largeNbDicts.c new file mode 100644 index 000000000..749d9660d --- /dev/null +++ b/contrib/largeNbDicts/largeNbDicts.c @@ -0,0 +1,289 @@ +/* + * Copyright (c) 2018-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +/* largeNbDicts + * This is a benchmark test tool + * dedicated to the specific case of dictionary decompression + * using a very large nb of dictionaries + * thus generating many cache-misses. + * It's created in a bid to investigate performance and find optimizations. */ + + +/*--- Dependencies ---*/ + +#include /* size_t */ +#include /* malloc, free */ +#include /* printf */ +#include /* assert */ + +#include "util.h" +#define ZSTD_STATIC_LINKING_ONLY +#include "zstd.h" +#include "zdict.h" + + +/*--- Constants --- */ + +#define KB *(1<<10) +#define MB *(1<<20) + +#define BLOCKSIZE (4 KB) +#define DICTSIZE (4 KB) +#define COMP_LEVEL 3 + +#define DISPLAY_LEVEL_DEFAULT 3 + + +/*--- Display Macros ---*/ + +#define DISPLAY(...) fprintf(stdout, __VA_ARGS__) +#define DISPLAYLEVEL(l, ...) { if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); } } +static int g_displayLevel = DISPLAY_LEVEL_DEFAULT; /* 0 : no display, 1: errors, 2 : + result + interaction + warnings, 3 : + progression, 4 : + information */ + + +/*--- buffer_t ---*/ + +typedef struct { + void* ptr; + size_t size; + size_t capacity; +} buffer_t; + +static const buffer_t kBuffNull = { NULL, 0, 0 }; + + +static buffer_t fillBuffer_fromHandle(buffer_t buff, FILE* f) +{ + size_t const readSize = fread(buff.ptr, 1, buff.capacity, f); + buff.size = readSize; + return buff; +} + +static void freeBuffer(buffer_t buff) +{ + free(buff.ptr); +} + +/* @return : kBuffNull if any error */ +static buffer_t createBuffer_fromHandle(FILE* f, size_t bufferSize) +{ + void* const buffer = malloc(bufferSize); + if (buffer==NULL) return kBuffNull; + + { buffer_t buff = { buffer, 0, bufferSize }; + buff = fillBuffer_fromHandle(buff, f); + if (buff.size != buff.capacity) { + freeBuffer(buff); + return kBuffNull; + } + return buff; + } +} + +/* @return : kBuffNull if any error */ +static buffer_t createBuffer_fromFile(const char* fileName) +{ + U64 const fileSize = UTIL_getFileSize(fileName); + size_t const bufferSize = (size_t) fileSize; + + if (fileSize == UTIL_FILESIZE_UNKNOWN) return kBuffNull; + assert((U64)bufferSize == fileSize); /* check overflow */ + + { buffer_t buff; + FILE* const f = fopen(fileName, "rb"); + if (f == NULL) return kBuffNull; + + buff = createBuffer_fromHandle(f, bufferSize); + fclose(f); /* do nothing specific if fclose() fails */ + return buff; + } +} + + +/*--- buffer_collection_t ---*/ + +typedef struct { + void** buffers; + size_t* capacities; + size_t nbBuffers; +} buffer_collection_t; + +static const buffer_collection_t kNullCollection = { NULL, NULL, 0 }; + +static void freeCollection(buffer_collection_t collection) +{ + free(collection.buffers); + free(collection.capacities); +} + +/* returns .buffers=NULL if operation fails */ +buffer_collection_t splitBuffer(buffer_t srcBuffer, size_t blockSize) +{ + size_t const nbBlocks = (srcBuffer.size + (blockSize-1)) / blockSize; + + void** const buffers = malloc(nbBlocks * sizeof(void*)); + size_t* const capacities = malloc(nbBlocks * sizeof(size_t*)); + if ((buffers==NULL) || capacities==NULL) { + free(buffers); + free(capacities); + return kNullCollection; + } + + char* newBlockPtr = (char*)srcBuffer.ptr; + char* const srcEnd = newBlockPtr + srcBuffer.size; + assert(nbBlocks >= 1); + for (size_t blockNb = 0; blockNb < nbBlocks-1; blockNb++) { + buffers[blockNb] = newBlockPtr; + capacities[blockNb] = blockSize; + newBlockPtr += blockSize; + } + + /* last block */ + assert(newBlockPtr <= srcEnd); + size_t const lastBlockSize = (srcEnd - newBlockPtr); + buffers[nbBlocks-1] = newBlockPtr; + capacities[nbBlocks-1] = lastBlockSize; + + buffer_collection_t result; + result.buffers = buffers; + result.capacities = capacities; + result.nbBuffers = nbBlocks; + return result; +} + + + +/*--- ddict_collection_t ---*/ + +typedef struct { + ZSTD_DDict** ddicts; + size_t nbDDict; +} ddict_collection_t; + +static const ddict_collection_t kNullDDictCollection = { NULL, 0 }; + +static void freeDDictCollection(ddict_collection_t ddictc) +{ + for (size_t dictNb=0; dictNb < ddictc.nbDDict; dictNb++) { + ZSTD_freeDDict(ddictc.ddicts[dictNb]); + } + free(ddictc.ddicts); +} + +/* returns .buffers=NULL if operation fails */ +static ddict_collection_t createDDictCollection(const void* dictBuffer, size_t dictSize, size_t nbDDict) +{ + ZSTD_DDict** const ddicts = malloc(nbDDict * sizeof(ZSTD_DDict*)); + if (ddicts==NULL) return kNullDDictCollection; + for (size_t dictNb=0; dictNb < nbDDict; dictNb++) { + ddicts[dictNb] = ZSTD_createDDict(dictBuffer, dictSize); + assert(ddicts[dictNb] != NULL); + } + ddict_collection_t ddictc; + ddictc.ddicts = ddicts; + ddictc.nbDDict = nbDDict; + return ddictc; +} + + + +/*--- Benchmark --- */ + + +/* bench() : + * @return : 0 is success, 1+ otherwise */ +int bench(const char* fileName) +{ + int result = 0; + + DISPLAYLEVEL(3, "loading %s... \n", fileName); + buffer_t const srcBuffer = createBuffer_fromFile(fileName); + if (srcBuffer.ptr == NULL) { + DISPLAYLEVEL(1," error reading file %s \n", fileName); + return 1; + } + DISPLAYLEVEL(3, "created src buffer of size %.1f MB \n", + (double)(srcBuffer.size) / (1 MB)); + + buffer_collection_t const srcBlockBuffers = splitBuffer(srcBuffer, BLOCKSIZE); + assert(srcBlockBuffers.buffers != NULL); + unsigned const nbBlocks = (unsigned)srcBlockBuffers.nbBuffers; + DISPLAYLEVEL(3, "splitting input into %u blocks of max size %u bytes \n", + nbBlocks, BLOCKSIZE); + + size_t const dstBlockSize = ZSTD_compressBound(BLOCKSIZE); + size_t const dstBufferCapacity = nbBlocks * dstBlockSize; + void* const dstPtr = malloc(dstBufferCapacity); + assert(dstPtr != NULL); + buffer_t dstBuffer; + dstBuffer.ptr = dstPtr; + dstBuffer.capacity = dstBufferCapacity; + dstBuffer.size = dstBufferCapacity; + + buffer_collection_t const dstBlockBuffers = splitBuffer(dstBuffer, dstBlockSize); + assert(dstBlockBuffers.buffers != NULL); + + DISPLAYLEVEL(3, "creating dictionary, of target size %u bytes \n", DICTSIZE); + void* const dictBuffer = malloc(DICTSIZE); + if (dictBuffer == NULL) { result = 1; goto _cleanup; } + + size_t const dictSize = ZDICT_trainFromBuffer(dictBuffer, DICTSIZE, + srcBuffer.ptr, + srcBlockBuffers.capacities, + nbBlocks); + if (ZSTD_isError(dictSize)) { + DISPLAYLEVEL(1, "error creating dictionary \n"); + result = 1; + goto _cleanup; + } + + size_t const dictMem = ZSTD_estimateDDictSize(dictSize, ZSTD_dlm_byCopy); + size_t const allDictMem = dictMem * nbBlocks; + DISPLAYLEVEL(3, "generating %u dictionaries, using %.1f MB of memory \n", + nbBlocks, (double)allDictMem / (1 MB)); + + ZSTD_CDict* const cdict = ZSTD_createCDict(dictBuffer, dictSize, COMP_LEVEL); + do { + ddict_collection_t const dictionaries = createDDictCollection(dictBuffer, dictSize, nbBlocks); + assert(dictionaries.ddicts != NULL); + + freeDDictCollection(dictionaries); + } while(0); + ZSTD_freeCDict(cdict); + +_cleanup: + free(dictBuffer); + freeCollection(dstBlockBuffers); + freeBuffer(dstBuffer); + freeCollection(srcBlockBuffers); + freeBuffer(srcBuffer); + + return result; +} + + + + +/*--- Command Line ---*/ + +int bad_usage(const char* exeName) +{ + DISPLAY (" bad usage : \n"); + DISPLAY (" %s filename \n", exeName); + return 1; +} + +int main (int argc, const char** argv) +{ + const char* const exeName = argv[0]; + + if (argc != 2) return bad_usage(exeName); + return bench(argv[1]); +} diff --git a/lib/Makefile b/lib/Makefile index 01689c6d5..cf8e45b0f 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -23,7 +23,7 @@ ifeq ($(OS),Windows_NT) # MinGW assumed CPPFLAGS += -D__USE_MINGW_ANSI_STDIO # compatibility with %zu formatting endif CFLAGS ?= -O3 -DEBUGFLAGS = -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \ +DEBUGFLAGS= -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \ -Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \ -Wstrict-prototypes -Wundef -Wpointer-arith -Wformat-security \ -Wvla -Wformat=2 -Winit-self -Wfloat-equal -Wwrite-strings \ diff --git a/lib/dictBuilder/zdict.h b/lib/dictBuilder/zdict.h index 4094669d1..3b3a6527c 100644 --- a/lib/dictBuilder/zdict.h +++ b/lib/dictBuilder/zdict.h @@ -52,7 +52,8 @@ extern "C" { * It's recommended that total size of all samples be about ~x100 times the target size of dictionary. */ ZDICTLIB_API size_t ZDICT_trainFromBuffer(void* dictBuffer, size_t dictBufferCapacity, - const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples); + const void* samplesBuffer, + const size_t* samplesSizes, unsigned nbSamples); /*====== Helper functions ======*/ diff --git a/programs/bench.h b/programs/bench.h index f6a53fc6a..184cc3e00 100644 --- a/programs/bench.h +++ b/programs/bench.h @@ -233,7 +233,7 @@ typedef size_t (*BMK_initFn_t)(void* initPayload); * srcSizes - an array of the sizes of above buffers * dstBuffers - an array of buffers to be written into by benchFn * dstCapacities - an array of the capacities of above buffers - * blockResults - store the return value of benchFn for each block. Optional. Use NULL if this result is not requested. + * blockResults - Optional: store the return value of benchFn for each block. Use NULL if this result is not requested. * nbLoops - defines number of times benchFn is run. * @return: a variant, which express either an error, or can generate a valid BMK_runTime_t result. * Use BMK_isSuccessful_runOutcome() to check if function was successful.