From 2ddf7e9a2829a2b7c950e4b93ea5e851bb858ae6 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Sun, 8 Feb 2015 20:26:47 +0100 Subject: [PATCH 1/2] Fixed : FSE issue with corner case distributions --- lib/fse.c | 49 ++++++------------------------------------------- lib/zstd.c | 2 -- 2 files changed, 6 insertions(+), 45 deletions(-) diff --git a/lib/fse.c b/lib/fse.c index a1aeea529..1f61a0a46 100644 --- a/lib/fse.c +++ b/lib/fse.c @@ -560,45 +560,9 @@ int FSE_compareRankT(const void* r1, const void* r2) return 2 * (R1->count < R2->count) - 1; } -#if 0 -static void FSE_adjustNormSlow(short* norm, int pointsToRemove, const unsigned* count, U32 maxSymbolValue) -{ - rank_t rank[FSE_MAX_SYMBOL_VALUE+1]; - U32 s; - - /* Init */ - for (s=0; s<=maxSymbolValue; s++) - { - rank[s].id = s; - rank[s].count = count[s]; - if (norm[s] <= 1) rank[s].count = 0; - } - - /* Sort according to count */ - qsort(rank, maxSymbolValue+1, sizeof(rank_t), FSE_compareRankT); - - while(pointsToRemove) - { - int newRank = 1; - norm[rank[0].id]--; - rank[0].count = (rank[0].count * 3) >> 2; - if (norm[rank[0].id] == 1) rank[0].count = 0; - while (rank[newRank].count > rank[newRank-1].count) - { - rank_t r = rank[newRank-1]; - rank[newRank-1] = rank[newRank]; - rank[newRank] = r; - newRank++; - } - pointsToRemove--; - } -} - -#else - static size_t FSE_adjustNormSlow(short* norm, int pointsToRemove, const unsigned* count, U32 maxSymbolValue) { - rank_t rank[FSE_MAX_SYMBOL_VALUE+1]; + rank_t rank[FSE_MAX_SYMBOL_VALUE+2]; U32 s; /* Init */ @@ -608,6 +572,8 @@ static size_t FSE_adjustNormSlow(short* norm, int pointsToRemove, const unsigned rank[s].count = count[s]; if (norm[s] <= 1) rank[s].count = 0; } + rank[maxSymbolValue+1].id = 0; + rank[maxSymbolValue+1].count = 0; /* ensures comparison ends here in worst case */ /* Sort according to count */ qsort(rank, maxSymbolValue+1, sizeof(rank_t), FSE_compareRankT); @@ -634,7 +600,6 @@ static size_t FSE_adjustNormSlow(short* norm, int pointsToRemove, const unsigned return 0; } -#endif size_t FSE_normalizeCount (short* normalizedCounter, unsigned tableLog, @@ -676,8 +641,7 @@ size_t FSE_normalizeCount (short* normalizedCounter, unsigned tableLog, short proba = (short)((count[s]*step) >> scale); if (proba<8) { - U64 restToBeat; - restToBeat = vStep * rtbTable[proba]; + U64 restToBeat = vStep * rtbTable[proba]; proba += (count[s]*step) - ((U64)proba< restToBeat; } if (proba > largestP) @@ -691,9 +655,8 @@ size_t FSE_normalizeCount (short* normalizedCounter, unsigned tableLog, } if (-stillToDistribute >= (normalizedCounter[largest] >> 1)) { - size_t errorCode; /* corner case, need to converge towards normalization with caution */ - errorCode = FSE_adjustNormSlow(normalizedCounter, -stillToDistribute, count, maxSymbolValue); + size_t errorCode = FSE_adjustNormSlow(normalizedCounter, -stillToDistribute, count, maxSymbolValue); if (FSE_isError(errorCode)) return errorCode; //FSE_adjustNormSlow(normalizedCounter, -stillToDistribute, count, maxSymbolValue); } @@ -1247,7 +1210,7 @@ size_t FSE_decompress(void* dst, size_t maxDstSize, const void* cSrc, size_t cSr const BYTE* const istart = (const BYTE*)cSrc; const BYTE* ip = istart; short counting[FSE_MAX_SYMBOL_VALUE+1]; - FSE_decode_t DTable[FSE_MAX_TABLESIZE]; + FSE_decode_t DTable[FSE_DTABLE_SIZE_U32(FSE_MAX_TABLELOG)]; unsigned maxSymbolValue = FSE_MAX_SYMBOL_VALUE; unsigned tableLog; size_t errorCode, fastMode; diff --git a/lib/zstd.c b/lib/zstd.c index 48c4025b1..6d5ceeca5 100644 --- a/lib/zstd.c +++ b/lib/zstd.c @@ -924,14 +924,12 @@ static const BYTE* ZSTD_updateMatch(U32* table, const BYTE* p, const BYTE* start static int ZSTD_checkMatch(const BYTE* match, const BYTE* ip) { - //return *(U32*)match == *(U32*)ip; return ZSTD_read32(match) == ZSTD_read32(ip); } static size_t ZSTD_compressBlock(void* ctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize) { - // Local Variables cctxi_t* srt = (cctxi_t*) ctx; U32* HashTable = (U32*)(srt->hashTable); void* workplace = srt->workplace; From 5ca33779a48ee44a1dffe29eadf274f3aac41a28 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Sun, 8 Feb 2015 21:51:28 +0100 Subject: [PATCH 2/2] Visual : Removed *.suo and *.user files, as suggested by Takayuki Matsuoka --- .gitignore | 4 ++++ visual/2012/fullbench/fullbench.vcxproj.user | 4 ---- visual/2012/fuzzer/fuzzer.vcxproj.user | 4 ---- visual/2012/zstd.sln | 14 -------------- visual/2012/zstd.v11.suo | Bin 23040 -> 0 bytes visual/2012/zstd/zstd.vcxproj.user | 4 ---- 6 files changed, 4 insertions(+), 26 deletions(-) delete mode 100644 visual/2012/fullbench/fullbench.vcxproj.user delete mode 100644 visual/2012/fuzzer/fuzzer.vcxproj.user mode change 100644 => 100755 visual/2012/zstd.sln delete mode 100644 visual/2012/zstd.v11.suo delete mode 100644 visual/2012/zstd/zstd.vcxproj.user diff --git a/.gitignore b/.gitignore index 9fa3b1bf7..c4c9282bd 100644 --- a/.gitignore +++ b/.gitignore @@ -16,3 +16,7 @@ *.exe *.out *.app + +# Visual solution files +*.suo +*.user diff --git a/visual/2012/fullbench/fullbench.vcxproj.user b/visual/2012/fullbench/fullbench.vcxproj.user deleted file mode 100644 index 7cbb3216a..000000000 --- a/visual/2012/fullbench/fullbench.vcxproj.user +++ /dev/null @@ -1,4 +0,0 @@ - - - - \ No newline at end of file diff --git a/visual/2012/fuzzer/fuzzer.vcxproj.user b/visual/2012/fuzzer/fuzzer.vcxproj.user deleted file mode 100644 index 7cbb3216a..000000000 --- a/visual/2012/fuzzer/fuzzer.vcxproj.user +++ /dev/null @@ -1,4 +0,0 @@ - - - - \ No newline at end of file diff --git a/visual/2012/zstd.sln b/visual/2012/zstd.sln old mode 100644 new mode 100755 index 95eb56ed4..2b5eb9895 --- a/visual/2012/zstd.sln +++ b/visual/2012/zstd.sln @@ -9,46 +9,32 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "fullbench", "fullbench\full EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution - Debug|Mixed Platforms = Debug|Mixed Platforms Debug|Win32 = Debug|Win32 Debug|x64 = Debug|x64 - Release|Mixed Platforms = Release|Mixed Platforms Release|Win32 = Release|Win32 Release|x64 = Release|x64 EndGlobalSection GlobalSection(ProjectConfigurationPlatforms) = postSolution - {4E52A41A-F33B-4C7A-8C36-A1A6B4F4277C}.Debug|Mixed Platforms.ActiveCfg = Debug|Win32 - {4E52A41A-F33B-4C7A-8C36-A1A6B4F4277C}.Debug|Mixed Platforms.Build.0 = Debug|Win32 {4E52A41A-F33B-4C7A-8C36-A1A6B4F4277C}.Debug|Win32.ActiveCfg = Debug|Win32 {4E52A41A-F33B-4C7A-8C36-A1A6B4F4277C}.Debug|Win32.Build.0 = Debug|Win32 {4E52A41A-F33B-4C7A-8C36-A1A6B4F4277C}.Debug|x64.ActiveCfg = Debug|x64 {4E52A41A-F33B-4C7A-8C36-A1A6B4F4277C}.Debug|x64.Build.0 = Debug|x64 - {4E52A41A-F33B-4C7A-8C36-A1A6B4F4277C}.Release|Mixed Platforms.ActiveCfg = Release|Win32 - {4E52A41A-F33B-4C7A-8C36-A1A6B4F4277C}.Release|Mixed Platforms.Build.0 = Release|Win32 {4E52A41A-F33B-4C7A-8C36-A1A6B4F4277C}.Release|Win32.ActiveCfg = Release|Win32 {4E52A41A-F33B-4C7A-8C36-A1A6B4F4277C}.Release|Win32.Build.0 = Release|Win32 {4E52A41A-F33B-4C7A-8C36-A1A6B4F4277C}.Release|x64.ActiveCfg = Release|x64 {4E52A41A-F33B-4C7A-8C36-A1A6B4F4277C}.Release|x64.Build.0 = Release|x64 - {6FD4352B-346C-4703-96EA-D4A8B9A6976E}.Debug|Mixed Platforms.ActiveCfg = Debug|Win32 - {6FD4352B-346C-4703-96EA-D4A8B9A6976E}.Debug|Mixed Platforms.Build.0 = Debug|Win32 {6FD4352B-346C-4703-96EA-D4A8B9A6976E}.Debug|Win32.ActiveCfg = Debug|Win32 {6FD4352B-346C-4703-96EA-D4A8B9A6976E}.Debug|Win32.Build.0 = Debug|Win32 {6FD4352B-346C-4703-96EA-D4A8B9A6976E}.Debug|x64.ActiveCfg = Debug|x64 {6FD4352B-346C-4703-96EA-D4A8B9A6976E}.Debug|x64.Build.0 = Debug|x64 - {6FD4352B-346C-4703-96EA-D4A8B9A6976E}.Release|Mixed Platforms.ActiveCfg = Release|Win32 - {6FD4352B-346C-4703-96EA-D4A8B9A6976E}.Release|Mixed Platforms.Build.0 = Release|Win32 {6FD4352B-346C-4703-96EA-D4A8B9A6976E}.Release|Win32.ActiveCfg = Release|Win32 {6FD4352B-346C-4703-96EA-D4A8B9A6976E}.Release|Win32.Build.0 = Release|Win32 {6FD4352B-346C-4703-96EA-D4A8B9A6976E}.Release|x64.ActiveCfg = Release|x64 {6FD4352B-346C-4703-96EA-D4A8B9A6976E}.Release|x64.Build.0 = Release|x64 - {61ABD629-1CC8-4FD7-9281-6B8DBB9D3DF8}.Debug|Mixed Platforms.ActiveCfg = Debug|Win32 - {61ABD629-1CC8-4FD7-9281-6B8DBB9D3DF8}.Debug|Mixed Platforms.Build.0 = Debug|Win32 {61ABD629-1CC8-4FD7-9281-6B8DBB9D3DF8}.Debug|Win32.ActiveCfg = Debug|Win32 {61ABD629-1CC8-4FD7-9281-6B8DBB9D3DF8}.Debug|Win32.Build.0 = Debug|Win32 {61ABD629-1CC8-4FD7-9281-6B8DBB9D3DF8}.Debug|x64.ActiveCfg = Debug|x64 {61ABD629-1CC8-4FD7-9281-6B8DBB9D3DF8}.Debug|x64.Build.0 = Debug|x64 - {61ABD629-1CC8-4FD7-9281-6B8DBB9D3DF8}.Release|Mixed Platforms.ActiveCfg = Release|Win32 - {61ABD629-1CC8-4FD7-9281-6B8DBB9D3DF8}.Release|Mixed Platforms.Build.0 = Release|Win32 {61ABD629-1CC8-4FD7-9281-6B8DBB9D3DF8}.Release|Win32.ActiveCfg = Release|Win32 {61ABD629-1CC8-4FD7-9281-6B8DBB9D3DF8}.Release|Win32.Build.0 = Release|Win32 {61ABD629-1CC8-4FD7-9281-6B8DBB9D3DF8}.Release|x64.ActiveCfg = Release|x64 diff --git a/visual/2012/zstd.v11.suo b/visual/2012/zstd.v11.suo deleted file mode 100644 index 40882be5a0345e12edab5795101bdfe385401ec4..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 23040 zcmca`Uhu)fjZzO8(10BSGsD0CoD6J8;*3aa1_1^J5e5bZkl4Tf|Nn!eK8C)4EAQD89Vhl`h`$6s?Mq36011R}|m4+}RG88jpGx#uMG88kE zFoZG`Go&&UF~G!&!K!c*Dhx~vd<+Z>;tUL6ci>iukV#=+VBmmq!x=Ie@)%MW@)^n* zd>IlM@)!~s(!nm!V+a7tUn0|Nsn z4nR%-#o1qwMm(5{fq_8`>PKD%2C#qmpmfq_u|hnYt%jW7S} zK+6+y%^uZ<83LgA1z~1r*n;957T2J307@4ic~E)))fpf*D7~YX{~&Qtdk|LUgX%+2 zJpjt)pz>dufq_AWfq_Amfq_Aefq_Aufq_ARfq_Ahfq_AZfq_98T7Q7zUyXr*K^Fz7KbFz7QdFc>f}Fc>m0Fc>i~Fc>p1FqkkfFqkqhFqkng zFqktiFjzqSWW~V1V9mh5V8g(`V9UV3V8_6~U=M18FfcGULe+rW?*e7JF)%Q=GcYiC zFfcH9GB7ZBLB&C8d>I%R{E+y83=9lG3=9mx3=9k*3=9mR3=9lm3=9n63=9kr3=9mB z3=9lW3=9m>P(85>3=DA$3=HuM3=9bj3=D}-agf=_7&E45BLXXbQyG#ON*F>IiWo}4 z?I2ittcW3(A(5ekA(J7WArDkwqG1CDCPo$p1_n_20l5%O8Y>s%Pnbj)gENByycU4C z0wzZcEylpa2!M*A1!bx<1%WV8!I zDnk-ODMLE6)#=NS%8<*D5AFvbw@p!7ogkeMtONBws0|1T1BfVo(ixgpK&sH~cVZ}F zNM%T5$Yv;D$Op$t2}3ahvP(d^5ZHta|2rYIzhUVg)bA)@$N~Et)c!}L0)$=ISfKa^ zm3yGFjadJK>QFyWBrq_5`#UAz_CBcPpTpqBkjan(Zso&V2FVl7;QR+t3F@UGV#A6I z|0Cx=XNDYb52%fy;DI8BeVLk?jv<$ODIZB0~zepM&aa zgg>#{4yrpy@Bbm&@4=ACkOHLhar!_jUk^Q2P{&A*4M)31|jk@H^&xU4HdDd9mQAD~jH5?n8XMrB|*!J8o! zZF~we{sAgaL3V>Ms!iB=F$@e0rO@&hyZu27pqvOwi6!8+PaZ=$LovK;Ms|e_0}~^t zEex_7gt6O*DjLARz;F@N2w-4f0EIDf{TmL>^T`Yu;BZ0JhmEJez{Ch@V}r(Lh_&CJ z0Wx+AG7sH;?Da25C3dXAz{CjZ8-UvS#M{??{O7`u4=(9JBa5Jx1#(L(fFYj&R3^Y$|JY&z)cyzg z8-`I`hn;5$>aT$!4V?aQ+YcLyh2?&1rs**-fya{tpkssB)S!zY`yUj~IpCTO(!+tY ze$p5+8PdV=kFW0oGFOL;`WKW}7l9HCxc`yFkj3B&uKyu+BieXj;2tc<7Tg$A|AEG3 zK>cn|9N<=okO9?~u>2Qs9+Yzsa-_0UFnTb>3Gg7-GO})Le!Fa9^PqoJK(XuaAq)OFk=Ae2dOe+aAGiFaAPoGFk&!gFlTUPsAaHa2x3SD zx8y3>6G!;1LRlO=b*k3@!{N48~ylK;|1Wm@t?zID>66XE0zeX3%A@WH4iJ z1=|Ty>&Rfi;KX3b;K*RcV98((mLtX{kQ<#CTo}w4j2J8#bQug8oEe-MEWkE_TwxBj z$%w&%!H_|h0puGC1{Ve=uw5<;#tbeDZVVQ9{lpODXZ^X#r^jsJvMpcKYaqc3nUv5r zy^^0}!t6fn)s+=Hmrc(@meahpnvsEli$Q7$+4>Vt$XxfI)R^77d#>#dWV6u|9l9Op z@}MCM4p{nOVBkq(C}pT(r~)^P7#M^=+#Ci_8x_(z0?i@qg^nqK>OV#@^@E%Y!t)qu zmj^&SyJCh?hD3%Oh8PA=`2uQF88XB$fP4k>Bq~pjp^PD!p#q%8K=X4TH==|IXrK!u z=mE;<$XFMg_dz1~Ss)e27!)DGOT+nn82C;~<^17P_Mh;-{m1!3OF-2ngAc=V5!FlU zm6hGMJ$+cKo|Fq3>;sRxsc`>w)b2g#vGLZb^W^BaNE12q?~t6&lDR4CrpvvzB+Y)1 zt9=DY#W#1fJUjjXHwL7|V0M~IDz}&$nF4*;!@Wgi1rt(geK$$ z(8w`$-GCkzkT#exgAoHEw@^P`ki&y`cR*4#_3AX@QYEP0g+0eX#>_w|hl2q$n^4U_ zJ-1nd=MboAq8^U60-@N4iHfy79g!?LOBCcM$Nk>+M2=H+)>xU+1O0i(a_P%$;8dX$lTnywx+7MBqgS**q8~jCV-J) z9;1tuft7((wTY{#k)w&Bqpq8=v6HR|e*GmWdS%HK1x5K;HO^KszLg3AMfvGPiMa}H znK`M&F)pcPG0yq91x2aF#hLkeF<@JZi;`n@3`L@yde{$#lL2Ft-aRe$|2w~8e*XaB|B$NZA}dN*f1pY7#Ie@{68aTP1orBKPdM@_Cdhn1{Vz~(_wO;(GCjd|4Ckd z1ZoLVasHoV`$2w(PJb{A^!z`*^ikdy}@N!4EL zl30?cotc-CTH&AOlUQ8hnvB0wpRmkTsr zj_gjbB4hy)*q}A3l<5E93|isJz@X|@C1JL==^P1GgO%pdZrC2QC=q??%BrEoGSsQk zt#El4XLXe9H;@}35sx52VnpLTHr{uP-iJ59{12KLfXr}!)<32)lo7f9gOv4u$mJjQ z^%nu)RcS=7{{XK^2c<4h5D?>WYOntQC3KM8AWV$?gL(Z28S6j6`<_642Vr9TPv!NW zB-cNq>pv0gl+pQr(8MhR1E|kU%~_ohHvbP^sSTR3VqjnZEt^$fC`MX9Rm`vvx>Xod zRZl~n>K-)n|5@OrAn+9c#7=L6_LP9yGo$nWu(^Evv;GKKn0goutvRr)w7?e^!3+aE z|4-TafzkPY(0*%TXB#0ip43~VMgEK@HQj(d^FsWb4rFeLnr=ai$JIdBWP?g|Li25- z^Z%5r;G=Y=6|{zr8Y>CO&kfX^bEh;nP}41xgs!B|0AJ^l<0^$b8zN5Q!? zr=+wf)iy7+w4^98M@J!`G$|)D**mo|BtJVf&o;@xIL$Q89Hst%?c>Ii)b%{`N>Yo` z5|dMlVW~dP7G^6b{S9pXFDc2)OLt5z$t+7npGH-(2`bG46$FlHC8pgL@tqJn0J_^#o1azzpI6rDWpE4@5|k7k>-|1dA%X@khF?tRSD-5VjSm z)D0@l3r+>c6L`8Dv_?|J#JQ`iM_IJN!obGJ=twwgxZ-cEff^>@)gKHDtbhLh|4+Gzpe&2M{l^K)jDjEnWGgQ- zXf6g;4iZI!j2@u!4_p#i5)N*pk_e;yXX9Dyn3DslBZ^^i!Kpc^$t9^6RRz|fWZ1<& NiS^e&mVbjG{sFfsWv&1K diff --git a/visual/2012/zstd/zstd.vcxproj.user b/visual/2012/zstd/zstd.vcxproj.user deleted file mode 100644 index 7cbb3216a..000000000 --- a/visual/2012/zstd/zstd.vcxproj.user +++ /dev/null @@ -1,4 +0,0 @@ - - - - \ No newline at end of file