From 274b60e6e6571ccce5982e36ac82eabe606c1364 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Mon, 27 Aug 2018 17:08:44 -0700 Subject: [PATCH] largeNbDicts can compress and compare dict vs noDict --- contrib/largeNbDicts/Makefile | 2 + contrib/largeNbDicts/largeNbDicts | Bin 13626 -> 14034 bytes contrib/largeNbDicts/largeNbDicts.c | 144 +++++++++++++++++++++------- 3 files changed, 114 insertions(+), 32 deletions(-) diff --git a/contrib/largeNbDicts/Makefile b/contrib/largeNbDicts/Makefile index 082f0102a..026d76f12 100644 --- a/contrib/largeNbDicts/Makefile +++ b/contrib/largeNbDicts/Makefile @@ -21,6 +21,8 @@ CFLAGS += $(DEBUGFLAGS) $(MOREFLAGS) default: largeNbDicts +all : largeNbDicts + largeNbDicts: LDFLAGS += -lzstd largeNbDicts: largeNbDicts.c $(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@ diff --git a/contrib/largeNbDicts/largeNbDicts b/contrib/largeNbDicts/largeNbDicts index 40416f050fa594cb085444ebb1ca433629d59318..c057a2b78aa551a2de831b6e304f8747a6ea3d0f 100755 GIT binary patch literal 14034 zcmX^A>+L^w1_nlE28ISE1_lNJ1_p)+tPBjT39lG3DNC=b(pEm9Ek>YyrMd?=TJ18N?^eIWDV zGg5O3Qj4&-k3||{-Xo|1AU-JEpiom1pLq#AoKE<%9XC?wcYHF)sn4 zodLv0Hv=jKragc@%1 zI6#U)Sb>27rWeEo#iuBU0mbq0c{%aLmAOgzIq?N0MGW!rsP5x{x(}3pKw3b2bo0bO z5>Ol;pOc8sJPD|IE1>E@d}Q-L{*{2rfhbV8LGr1Or=Pd0izh6P8K8xm0Z26i!xfMR z85kHq=791GM3jL+iGiU3ti*tU0TebI1`G_av;|UQV8FnzgOP#Zg&_k&1p@;EC@w+n zC@^ARkY`|E5HV$7IKarj0Lp#=APrD8Aax)sL1v0W#j#NdCJYR^SS0uu7{DbG4+8^( zera)$eokhReoAFd3RJB$0|Nud9d9k&nJ<5}jb4!?>6xKZt*!zs(-^oI7`Pa;U_7WU z4F-k=kggA);Dd^RD3C1``QgCH)Noua16I2VxS_Uqdf>B~L z1V%$(Gz3ONU^E0qLtr!nMniz25D4|?eCinP80Hx27!vH!{6@l~^Rq{1?FEl+R|$_! z*AqUyCCvW~FZpzS@c91Ov-7x5ugX!7g+86nUu@uJVDRib3Suq=GmrSTzAbV0ZGBSW zu$O zqXFd@fH($Fjs=Kg;lX&rqxlerNAs~CrTreQCrcDOdrhu*_J&+#nBdXL?0Cy8Qj&-C6VJ2go-*ofD16#^d3M0Hf}}rjGBBj2rRnj@ zw}68NC4K`uJI{M|Uh(Ms?AiIxqxH5&H|rlhP{>-8$a(adx^puy7#{HGy!B!qCn(&% zdvxA=v73Q`;dP8l=V6atQ#+6~p#G-effutm85s71*e_Ocf|CI#?%no*M0`5m`*c3? z>HO)~dEUnm?ad;kg6M!iH;L&`91J(OXV2_zL zf_=-&3bJ@D$bUj0<9cl$gT+mgz!dLKE(V5On?cf_Il$h>;{T9fkH$9>KUAFfj0MZw>kX|Gz532L=XP&^QTy>kN<{gzAG()s5Ixm*)9& zmu~QE{`bEm#iQ4>l?UYa`!6y%Kp}M8xAlLCuTSUq7v&rv@xvaiw@VcEf%;V*o%ek@ zKlyb2eBs5xz~E!~p@bLYGG`7323NyxhPQn>|G$s}8zQ6P+gYN*0U`umOL-o5Q2}QN z5CfbeJdd-0I(!TsjYmM91qCWNMPW@39*u8afHGp|H;>NWKHawOc^DWxyX`%CP4{!c zlEP+oP&01m8sFV_6~|KFp#6~c90;n8}4zhyNuSleg*y55Zc|Np1)C-F`N#RGo~ z?_4llbMh0v0P8|9TeBAw?w|N04|?>Pws0~qY+!uh53-}XwE%2~i;4iuf^M*R9-Tfa z93Gu4DjvOi75@MK4^Cq*oPUEI0uu1(bUgreh$Ay7G0!#l|NlS48=0I83_C$-%3sHL*)Oo;KwPlHo-u(U-1UM-_goE-alLyvz$(9a zbce3+=)CFE?YiMb;Gh5heY&@TWPDq{@wZ$6Yl=PL(^)&gr+cZu|NsAYfd)W)I=_SL zc)|)ca|hUh!%PeeKHYmAK!$fu^#D<@gwS1k0aWUChA!~v_C4U!?RvtaJ9dL-=gk)a zpy28}2r7147J@A6W_=6FP~8@39{hPHLGgH&je#MJKZf@zn65dQ#;?%|iN82d{4M2R zU;w*m1LKP>kbAmacYtFGY-#Nbkj32yi$Rvo23Z8Q)U)$Ge~Sl*+s#@EvbXcFN4GWH z>L9R{F}zV=8pG-a4p7o@>3sA;kd=YK@EfQWy34}A;K{%KDAbq(Y5Ym7++gi7tioWr z=D;WZNLFz+P?|QahZ^zg#W@xR2CrV5x1b{P#Z8bYoscwef{}p%RIqs(-cI9}cL5~` zaNy^H)v(+L2YwHzg!q5h$MSX21E0>{X~^ZTN4NEhAQlD&kLJT1p8wC5z6WLHY>+^A z?T!~lEMOB~L_q{DyaHe{93ou;kzNjw)&NQC zfTS%T(xDLP4v4e@NLn8xEeVk}gGgsWr1?S8Mj+{5pmfx0D-MzNf=K&-q(Nh;FK$Dm zQ$PhxucOqnUe+#H0GQ90-_}`4J7>%B<%%}PJ>9#f=JH;NxuO}Ycn!1fb$zTKEi^*^^>VT zsC4e;^XRqx#lpa_4>TgW3sil+07Zspx1C3?=?6%j=)Cpf7XxIRmLb|PF7_}iyujv# zJBEQ;yPcnXx{aIvGuEm4bXRk@8a`?M&r}lH{GX*%#G|`fz@yjnAPWP-i>nNv&?<58 z=$1VPa&9-bN3U%qssrYOihyowk6zP!umgNLKfS1AU|?|E19IPM0mE;vSv*KVu9NIFMB{NMn(pPmrWp+D2N5BO*1q>tUQpIIf#`4V!47?Q6N?*hy|*AGLk_o zH;`BXh-Cv})q_|@AXXoU1*&E<7K2!F|Ns97c?gCLoEaDlTwy#24H5&fVB#C>iVJcw zOPos6(o%~UauSQuQ~i=$GLuV+^^zI#lALn#le3Ez>}(Yb8S+xg!Q6n7A_W^;h2o-Q z*Sr*loczR;%)E4kl+5Ik%>2B>qDlqTVg)XSyEth5R&y5|9-o3dNaKsS2v4 z3Q3hEsc_|~Ad4Y}Fo4X1xJp6MR>3dS$A>{NDmcU?KC{@hs3^Zk1Ee50vno{+?gZ6h zJv}`IE{3GkyyOgq;-X}Te!YT{BDfh~XQifqT(6J>_B6;*5ZCA#rYZP3DR6{PIBo2a+&gNY2kKC`v6Z1_ga$i9$|lS!xc*edU=Y8Tq9p z$nj*Q5R_PwnQyDAXOyO(paF{wO;GH?%|E1NaTSW1&K5TJSL{6=A{-v zQiN(L!WEgR#X3+URiWXZpQezTnwwu#sldgckdz3rF)=+=!3xAxEmla&%t_5l%uQ8@ z)^$OcQyHtk#h~lLP?Q7@0dNQxGoXicUJ@i^ixq6ai5RR5Dx?ROWJpdfsQ~#MOUg?I zYXJqkCM21sx`4vpIX@>S6`TneQb1vXNG}j>G)T}dDHd)jXqJe9K?XW&2JXx-Fff2< z&^(kNHw)v%2v$Z0eg*~`1_%btNP}`Oh!3jmcog@=>RAl1*KDXdR=B1<-F@WY%8HywX z9pa;0Je@=0ONtUR^W2K^b0Nbd3=n6G{`Y9GdxgaU|<4|rzR(+#FrK)rl&GwFf)8m zWngAlV9dbG@WB{lCj$dRgWLn=2Bry23%DjQg9gs0GBPmmF)}bb5QL0#t%33dBp`f{ zCH#!wIeqX57HGvhKO+Of0!fIxB52tGBLl++AqZas%6}jL;afrZp!M9KfiBP*ZjgS^ zco~TA%K%mfVuD==T0PDP76Nm@E83Cy-Dv8UqVcz(@lT-fZ=vx&q47bZame<8rXi8} zpqU*;$Rc~N13|NF$b3tPVo{|x#GAL@BufPzGC7;A6!IOGCWG*2dBeTSVb4omNj>1t`#)D@q zP>V@aF>pzTDui4#LS+zzB8mVg12Z6tK#DJj2xbuo5l0k$2tKktSb-NGACFYjA=MkO z@-99NT+qd*K})#!G;q-dqEZ>+(;$UeJhT*xM=8GIi%Y=e7Dnv@4Qxm`6`xj=pX-*H Qlgbbe4?^%v3piB(0Dn~WwEzGB delta 3292 zcmcbVyDMwL1t9}b1_nk31_m{D1~4!Xo%qO3C`<+-ng$kQU26LS|`3k;!|Rt(k0CC;wu$atvT(U}!L8U`SwKU;r5>!N9=q z!jORhME@~nU~ph$VBlt8V1OA4HJgEzfdPz#L9SxWFlJy-ofs%Qc?FAxz>?dWjw}zf zS~6+dj;A(WQ`by>z@o;)ATybRwNL1QAp-*k0|SF50|SEq*s9G36>Rw@A7J&I{EyFu z*MXgZ!L#$IPv`R&rtFj5`9rxbu`w`!xTn}AFXR7cd7F)a!K2so0hr=_!N$O_s~2R> zLROHt?H;hW=|M2XdmSV_6(rrnI{AWtbiGI88x2MV29M6)KHa*PI2af_yK6zltz%&(Ru$x3M&Ic>jD1O7$yb={_U+-{{8>2%J6}K!Iq(cfq}o( zl?h}kT(ucgwH|iW;`K}n3_jgk4gUZC@7etCKYt4wSik8Jc95Iyzqrf-^8In&*8lu1 zuNgu9{{G@FSnP;z>yr|NeV}yZ)A`(|^OH~K&lek57#MskKa}u+!e=E51B0vKH&??a zKArzxG=ro%mx8?E)4BD=zyJSzI=4Qk|M&mDZ|f6~r} zt`j^uV;6XI#?J8Qc3t4n33i<81dndt8OY8``2WBDzfb447kpq>%}x0K|G!7;ZT=Pq zCU`n%W@TX51+v-jzzcu4D`Fw8@NNC((HXmgzhwpkC@6dzz@GH!{O;44dcddikw>TR z37>A)3okB%4D37tij_}Hp!8Ex?%D0j;L%%e8^p@MuuqzSfngUYDqs9#Vqn+}V!adq z#Rh-)fi!+iNVpyc2Q?_WK|@KN1(rs)!E|-r@Mt^&O3=}cF^;j0agOn^hr!1Abk{b# zJj%epVE7H>ifE97_}3q=2N{*-!Jl^k6z37lkT_2O(~vk%0R?Zb=?NB4oO^VKUU=cf z#K5qNnE{*_eLA1LV1b1D1+QM4zaZDWa0D6NdDOS{Eq{v`BLjnP>;DpC!`t9+dG_Kq zBPa%4?t&t%q`uDM|6w1?-$jpoI$wKqTfevf%8bp2IXwTLFMaRR`ToUAkU)3sg%=wb z85j(2`*f!s@ae8S;nVs3#W_%h=-!$Eini|B9WOwc0VKW+qGiL2wGhUN7fV1YJO96! z1E!z9n8L`w@bc2X|NrYfdToA2T)BFG!Pyjd&^47+xNN{Aqk?q1tpU~$tTFvSas zzFjv!(#8j6kw1R}i$EN*H7rg$Ym($7KCUqLCR*S3BxSi)2k zO!3YJ$^Qq*Ut?fk2=(ZE>KGObE}Km4LAk!0&!g9N5;FtCK0bJgxxfHQtacu~rX9?% z^stJ7fnf(I6-7J7#UAFDZvn*%L-T*8QcjQVI*yl2zyJRSn_eFl?9naf({1_?l%m1L zo1hwB1UFs_ZoChQ@&8r#Ff%axSKS9D4}i%-VDbo6P_mW?*m({;z7wJeglwsotM~f#Kze|NsAI zq=Q)dK&(~}YYT`q8^l@#Vr>Mm7Jyiw4DfOqh;x?{Qp108pL#){7||^LA6*$P@%xXz{DT|Za6S7Hn1@;C^9fHUW{O!{83)E{tUEj za}7#AfYPs^^cN`o4@z@Dn>9jES_Vq1L1_ahZ3U%WptK*9j)2lh5So=CN0xzM@<(~q z%@-8(KurumRzW3+1i{J72D$v8Fi&*`1rNgnh9ZN>=MBV}Ed(7VzcH0Cn7~lvAm|Vu z6&&IcpH`HLQ09Rw3JzP4=77nDhQjp`LJslq@rlL7sYNC6MJ0J4nFJw+__XAl{Nhv) zF9XEOFGvMT6@WMeMVWaeX&|`@5Eqn)K%53f2oJ2Q1I#He$}9nMCxE!Qi8(p>$snaO zK-}Vz#FBayPd|SbS2Gia2@DL3@eHgCEb+M+@u?N5$)zQ!@foR!DXB#ahTt^lRGOBS zTEt+Q#9)=1n3=~A28v~7h7DQ_3{2n@>ywxg4^CVRfy@jO^ca{KHkdOoGc;I0lE?Xnau7$p~rXf^E46kq3Dm%zuf-|A@-3 zXZV3Cz`)1|b~fAq4m3U=8Xr_UB0E41O&(OzAj^Z=*lY|844{q#2gBq96ERbMXuDaE zfq_AYfq_Aofq_AUfq_Akfq_Acfq?;3QHXl$i%gH4F?40Jqzgga7~l diff --git a/contrib/largeNbDicts/largeNbDicts.c b/contrib/largeNbDicts/largeNbDicts.c index 749d9660d..536e45ffe 100644 --- a/contrib/largeNbDicts/largeNbDicts.c +++ b/contrib/largeNbDicts/largeNbDicts.c @@ -159,6 +159,33 @@ buffer_collection_t splitBuffer(buffer_t srcBuffer, size_t blockSize) } +/*--- dictionary creation ---*/ + +buffer_t createDictionary(const char* dictionary, + const void* srcBuffer, size_t* srcBlockSizes, unsigned nbBlocks) +{ + if (dictionary) { + DISPLAYLEVEL(3, "loading dictionary %s \n", dictionary); + return createBuffer_fromFile(dictionary); + } else { + DISPLAYLEVEL(3, "creating dictionary, of target size %u bytes \n", DICTSIZE); + void* const dictBuffer = malloc(DICTSIZE); + assert(dictBuffer != NULL); + + size_t const dictSize = ZDICT_trainFromBuffer(dictBuffer, DICTSIZE, + srcBuffer, + srcBlockSizes, + nbBlocks); + assert(!ZSTD_isError(dictSize)); + + buffer_t result; + result.ptr = dictBuffer; + result.capacity = DICTSIZE; + result.size = dictSize; + return result; + } +} + /*--- ddict_collection_t ---*/ @@ -181,6 +208,7 @@ static void freeDDictCollection(ddict_collection_t ddictc) static ddict_collection_t createDDictCollection(const void* dictBuffer, size_t dictSize, size_t nbDDict) { ZSTD_DDict** const ddicts = malloc(nbDDict * sizeof(ZSTD_DDict*)); + assert(ddicts != NULL); if (ddicts==NULL) return kNullDDictCollection; for (size_t dictNb=0; dictNb < nbDDict; dictNb++) { ddicts[dictNb] = ZSTD_createDDict(dictBuffer, dictSize); @@ -193,29 +221,64 @@ static ddict_collection_t createDDictCollection(const void* dictBuffer, size_t d } +/* --- Compression --- */ -/*--- Benchmark --- */ +/* compressBlocks() : + * @return : total compressed size of all blocks, + * or 0 if error. + */ +static size_t compressBlocks(buffer_collection_t dstBlockBuffers, buffer_collection_t srcBlockBuffers, ZSTD_CDict* cdict, int cLevel) +{ + size_t const nbBlocks = srcBlockBuffers.nbBuffers; + assert(dstBlockBuffers.nbBuffers == srcBlockBuffers.nbBuffers); + + ZSTD_CCtx* const cctx = ZSTD_createCCtx(); + assert(cctx != NULL); + + size_t totalCSize = 0; + for (size_t blockNb=0; blockNb < nbBlocks; blockNb++) { + size_t cBlockSize; + if (cdict == NULL) { + cBlockSize = ZSTD_compressCCtx(cctx, + dstBlockBuffers.buffers[blockNb], dstBlockBuffers.capacities[blockNb], + srcBlockBuffers.buffers[blockNb], srcBlockBuffers.capacities[blockNb], + cLevel); + assert(!ZSTD_isError(cBlockSize)); + } else { + cBlockSize = ZSTD_compress_usingCDict(cctx, + dstBlockBuffers.buffers[blockNb], dstBlockBuffers.capacities[blockNb], + srcBlockBuffers.buffers[blockNb], srcBlockBuffers.capacities[blockNb], + cdict); + assert(!ZSTD_isError(cBlockSize)); + } + totalCSize += cBlockSize; + } + return totalCSize; +} + + +/* --- Benchmark --- */ /* bench() : + * fileName : file to load for benchmarking purpose + * dictionary : optional (can be NULL), file to load as dictionary, + * if none provided : will be calculated on the fly by the program. * @return : 0 is success, 1+ otherwise */ -int bench(const char* fileName) +int bench(const char* fileName, const char* dictionary) { int result = 0; DISPLAYLEVEL(3, "loading %s... \n", fileName); buffer_t const srcBuffer = createBuffer_fromFile(fileName); - if (srcBuffer.ptr == NULL) { - DISPLAYLEVEL(1," error reading file %s \n", fileName); - return 1; - } + assert(srcBuffer.ptr != NULL); DISPLAYLEVEL(3, "created src buffer of size %.1f MB \n", (double)(srcBuffer.size) / (1 MB)); buffer_collection_t const srcBlockBuffers = splitBuffer(srcBuffer, BLOCKSIZE); assert(srcBlockBuffers.buffers != NULL); unsigned const nbBlocks = (unsigned)srcBlockBuffers.nbBuffers; - DISPLAYLEVEL(3, "splitting input into %u blocks of max size %u bytes \n", + DISPLAYLEVEL(3, "split input into %u blocks of max size %u bytes \n", nbBlocks, BLOCKSIZE); size_t const dstBlockSize = ZSTD_compressBound(BLOCKSIZE); @@ -230,36 +293,44 @@ int bench(const char* fileName) buffer_collection_t const dstBlockBuffers = splitBuffer(dstBuffer, dstBlockSize); assert(dstBlockBuffers.buffers != NULL); - DISPLAYLEVEL(3, "creating dictionary, of target size %u bytes \n", DICTSIZE); - void* const dictBuffer = malloc(DICTSIZE); - if (dictBuffer == NULL) { result = 1; goto _cleanup; } + /* dictionary determination */ + buffer_t const dictBuffer = createDictionary(dictionary, + srcBuffer.ptr, + srcBlockBuffers.capacities, nbBlocks); + assert(dictBuffer.ptr != NULL); - size_t const dictSize = ZDICT_trainFromBuffer(dictBuffer, DICTSIZE, - srcBuffer.ptr, - srcBlockBuffers.capacities, - nbBlocks); - if (ZSTD_isError(dictSize)) { - DISPLAYLEVEL(1, "error creating dictionary \n"); - result = 1; - goto _cleanup; - } + ZSTD_CDict* const cdict = ZSTD_createCDict(dictBuffer.ptr, dictBuffer.size, COMP_LEVEL); + assert(cdict != NULL); - size_t const dictMem = ZSTD_estimateDDictSize(dictSize, ZSTD_dlm_byCopy); + size_t const cTotalSizeNoDict = compressBlocks(dstBlockBuffers, srcBlockBuffers, NULL, COMP_LEVEL); + assert(cTotalSizeNoDict != 0); + DISPLAYLEVEL(3, "compressing at level %u without dictionary : Ratio=%.2f (%u bytes) \n", + COMP_LEVEL, + (double)srcBuffer.size / cTotalSizeNoDict, (unsigned)cTotalSizeNoDict); + + size_t const cTotalSize = compressBlocks(dstBlockBuffers, srcBlockBuffers, cdict, COMP_LEVEL); + assert(cTotalSize != 0); + DISPLAYLEVEL(3, "compressed using a %u bytes dictionary : Ratio=%.2f (%u bytes) \n", + (unsigned)dictBuffer.size, + (double)srcBuffer.size / cTotalSize, (unsigned)cTotalSize); + + size_t const dictMem = ZSTD_estimateDDictSize(dictBuffer.size, ZSTD_dlm_byCopy); size_t const allDictMem = dictMem * nbBlocks; DISPLAYLEVEL(3, "generating %u dictionaries, using %.1f MB of memory \n", nbBlocks, (double)allDictMem / (1 MB)); - ZSTD_CDict* const cdict = ZSTD_createCDict(dictBuffer, dictSize, COMP_LEVEL); - do { - ddict_collection_t const dictionaries = createDDictCollection(dictBuffer, dictSize, nbBlocks); - assert(dictionaries.ddicts != NULL); + ddict_collection_t const dictionaries = createDDictCollection(dictBuffer.ptr, dictBuffer.size, nbBlocks); + assert(dictionaries.ddicts != NULL); - freeDDictCollection(dictionaries); - } while(0); + + + //result = benchMem(srcBlockBuffers, dstBlockBuffers, dictionaries);; + + + + freeDDictCollection(dictionaries); ZSTD_freeCDict(cdict); - -_cleanup: - free(dictBuffer); + freeBuffer(dictBuffer); freeCollection(dstBlockBuffers); freeBuffer(dstBuffer); freeCollection(srcBlockBuffers); @@ -276,7 +347,7 @@ _cleanup: int bad_usage(const char* exeName) { DISPLAY (" bad usage : \n"); - DISPLAY (" %s filename \n", exeName); + DISPLAY (" %s filename [-D dictionary] \n", exeName); return 1; } @@ -284,6 +355,15 @@ int main (int argc, const char** argv) { const char* const exeName = argv[0]; - if (argc != 2) return bad_usage(exeName); - return bench(argv[1]); + if (argc < 2) return bad_usage(exeName); + const char* const fileName = argv[1]; + + const char* dictionary = NULL; + if (argc > 2) { + if (argc != 4) return bad_usage(exeName); + if (strcmp(argv[2], "-D")) return bad_usage(exeName); + dictionary = argv[3]; + } + + return bench(fileName, dictionary); }