Merge pull request #61 from Cyan4973/dev

v0.3.2

Makefile

@@ -32,7 +32,7 @@
 # ################################################################
 
 # Version number
-export VERSION := 0.3.0
+export VERSION := 0.3.2
 
 PRGDIR  = programs
 ZSTDDIR = lib

NEWS

@@ -1,3 +1,9 @@
+v0.3.2
+Fixed Visual Studio
+
+v0.3.1 :
+Small compression ratio improvement
+
 v0.3
 HC mode : compression levels 2-26
 

README.md

@@ -13,36 +13,32 @@ For a taste of its performance, here are a few benchmark numbers from a number o
 |-----------------|-------|--------:|--------:|
 |                 |       |   MB/s  |  MB/s   |
 | **zstd 0.3**    |**2.858**|**280**| **670** |
-| [zlib 1.2.8] -1 | 2.730 |    70   |   300   | 
+| [zlib] 1.2.8 -1 | 2.730 |    70   |   300   | 
 | QuickLZ 1.5.1b6 | 2.237 |   370   |   415   |
 | LZO 2.06        | 2.106 |   400   |   580   |
 | [LZ4] r131      | 2.101 |   450   |  2100   |
 | Snappy 1.1.0    | 2.091 |   330   |  1100   |
 | LZF 3.6         | 2.077 |   200   |   560   |
 
-[zlib 1.2.8]:http://www.zlib.net/
+[zlib]:http://www.zlib.net/
 [LZ4]:http://www.lz4.org/
 
-Zstd can also offer stronger compression ratio at the cost of compression speed, but preserving its decompression speed. In the following test, a few compressors suitable for this scenario are selected (they offer very asymetric performance, useful when compression time has little importance). The test was completed on a Core i7-5600U @ 2.6 GHz, using [benchmark 0.6.1](http://encode.ru/threads/1266-In-memory-benchmark-with-fastest-LZSS-(QuickLZ-Snappy)-compressors?p=45217&viewfull=1#post45217), an open-source benchmark program by inikep.
+Zstd can also offer stronger compression ratio at the cost of compression speed. Speed / Ratio trade-off is configurable by small increment, to fit different situations. Note however that decompression speed is preserved and remain roughly the same at all settings, a property shared by most LZ compression algorithms, such as [zlib]. The following test is run on a Core i7-3930K CPU @ 4.5GHz, using [lzbench], an open-source in-memory benchmark by inikep, on the [Silesia compression corpus](http://sun.aei.polsl.pl/~sdeor/index.php?page=silesia)
 
-|Name             | Ratio | C.speed | D.speed |
-|-----------------|-------|--------:|--------:|
-|                 |       |   MB/s  |  MB/s   |
-| brotli -9       | 3.729 |     4   |   340   |
-| **zstd 0.3 -9** |**3.447**|**30** | **640** |
-| [zlib 1.2.8] -9 | 3.133 |    10   |   300   | 
-| LZO 2.06 -999   | 2.790 |     1   |   560   |
-| [LZ4] r131 -9   | 2.720 |    25   |  2100   |
+[lzbench]:https://github.com/inikep/lzbench
 
-[lzma]:http://www.7-zip.org/
+Compression Speed vs Ratio | Decompression Speed
+---------------------------|--------------------
+![Compression Speed vs Ratio](images/CSpeed.png "Compression Speed vs Ratio") | ![Decompression Speed](images/DSpeed.png "Decompression Speed")
 
-Zstd compression speed is highly configurable, by small increment, to fit different situations. Its memory requirement can also be configured to fit into low-memory hardware configurations, or servers handling multiple connections/contexts in parallel.
 
 Zstd entropy stage is provided by [Huff0 and FSE, from Finite State Entrop library](https://github.com/Cyan4973/FiniteStateEntropy).
 
-Zstd has not yet reached "stable" status. Specifically, it doesn't guarantee yet that its current compressed format will remain stable and supported in future versions. It may still change to adapt further optimizations still being investigated. That being said, the library is now pretty robust, able to withstand hazards situations, including invalid input. The library reliability has been tested using [Fuzz Testing](https://en.wikipedia.org/wiki/Fuzz_testing), with both [internal tools](programs/fuzzer.c) and [external ones](http://lcamtuf.coredump.cx/afl). Therefore, it's now safe to test Zstandard even within production environments.
+Its memory requirement can also be configured to fit into low-memory hardware configurations, or servers handling multiple connections/contexts in parallel.
 
-"Stable Format" is projected sometimes early 2016.
+Zstd has not yet reached "stable format" status. It doesn't guarantee yet that its current compressed format will remain stable and supported in future versions. During this period, it can still change to adapt new optimizations still being investigated. "Stable Format" is projected sometimes early 2016. 
+
+That being said, the library is now fairly robust, able to withstand hazards situations, including invalid inputs. The library reliability has been tested using [Fuzz Testing](https://en.wikipedia.org/wiki/Fuzz_testing), with both [internal tools](programs/fuzzer.c) and [external ones](http://lcamtuf.coredump.cx/afl). Therefore, it seems now safe to test Zstandard even within production environments.
 
 ### Branch Policy
 The "dev" branch is the one where all contributions will be merged before reaching "master". If you plan to propose a patch, please commit into the "dev" branch or its own feature branch. Direct commit to "master" are not permitted.

lib/huff0.c

@@ -362,7 +362,7 @@ size_t HUF_buildCTable (HUF_CElt* tree, const U32* count, U32 maxSymbolValue, U3
             U16 min = 0;
             for (n=maxNbBits; n>0; n--)
             {
-                valPerRank[n] = min;      // get starting value within each rank
+                valPerRank[n] = min;      /* get starting value within each rank */
                 min += nbPerRank[n];
                 min >>= 1;
             }
@@ -1027,18 +1027,23 @@ size_t HUF_readDTableX4 (U32* DTable, const void* src, size_t srcSize)
         U32 nextRankVal = 0;
         U32 w, consumed;
         const int rescale = (memLog-tableLog) - 1;   /* tableLog <= memLog */
+        U32* rankVal0 = rankVal[0];
         for (w=1; w<=maxW; w++)
         {
             U32 current = nextRankVal;
             nextRankVal += rankStats[w] << (w+rescale);
-            rankVal[0][w] = current;
+            rankVal0[w] = current;
+        }
+        for (consumed = minBits; consumed <= memLog - minBits; consumed++)
+        {
+            U32* rankValPtr = rankVal[consumed];
+            for (w = 1; w <= maxW; w++)
+            {
+                rankValPtr[w] = rankVal0[w] >> consumed;
+            }
         }
-        for (consumed=minBits; consumed <= memLog-minBits; consumed++)
-            for (w=1; w<=maxW; w++)
-                rankVal[consumed][w] = rankVal[0][w] >> consumed;
     }
 
-
     HUF_fillDTableX4(dt, memLog,
                    sortedSymbol, sizeOfSort,
                    rankStart0, rankVal, maxW,
@@ -1394,18 +1399,23 @@ size_t HUF_readDTableX6 (U32* DTable, const void* src, size_t srcSize)
         U32 nextRankVal = 0;
         U32 w, consumed;
         const int rescale = (memLog-tableLog) - 1;   /* tableLog <= memLog */
+        U32* rankVal0 = rankVal[0];
         for (w=1; w<=maxW; w++)
         {
             U32 current = nextRankVal;
             nextRankVal += rankStats[w] << (w+rescale);
-            rankVal[0][w] = current;
+            rankVal0[w] = current;
+        }
+        for (consumed = minBits; consumed <= memLog - minBits; consumed++)
+        {
+            U32* rankValPtr = rankVal[consumed];
+            for (w = 1; w <= maxW; w++)
+            {
+                rankValPtr[w] = rankVal0[w] >> consumed;
+            }
         }
-        for (consumed=minBits; consumed <= memLog-minBits; consumed++)
-            for (w=1; w<=maxW; w++)
-                rankVal[consumed][w] = rankVal[0][w] >> consumed;
     }
 
-
     /* fill tables */
     {
         HUF_DDescX6* DDescription = (HUF_DDescX6*)(DTable+1);

lib/zstd.h

@@ -48,7 +48,7 @@ extern "C" {
 ***************************************/
 #define ZSTD_VERSION_MAJOR    0    /* for breaking interface changes  */
 #define ZSTD_VERSION_MINOR    3    /* for new (non-breaking) interface capabilities */
-#define ZSTD_VERSION_RELEASE  0    /* for tweaks, bug-fixes, or development */
+#define ZSTD_VERSION_RELEASE  2    /* for tweaks, bug-fixes, or development */
 #define ZSTD_VERSION_NUMBER  (ZSTD_VERSION_MAJOR *100*100 + ZSTD_VERSION_MINOR *100 + ZSTD_VERSION_RELEASE)
 unsigned ZSTD_versionNumber (void);
 

lib/zstd_internal.h

@@ -49,6 +49,8 @@ extern "C" {
 ****************************************/
 static size_t ZSTD_read_ARCH(const void* p) { size_t r; memcpy(&r, p, sizeof(r)); return r; }
 
+#define MIN(a,b) ((a)<(b) ? (a) : (b))
+
 static unsigned ZSTD_highbit(U32 val)
 {
 #   if defined(_MSC_VER)   /* Visual */

lib/zstdhc.c

@@ -191,7 +191,7 @@ static size_t ZSTD_HC_hashPtr(const void* p, U32 h, U32 mls)
 *  HC Compression
 ***************************************/
 /* Update chains up to ip (excluded) */
-static void ZSTD_HC_insert (ZSTD_HC_CCtx* zc, const BYTE* ip, U32 mls)
+static U32 ZSTD_HC_insertAndFindFirstIndex  (ZSTD_HC_CCtx* zc, const BYTE* ip, U32 mls)
 {
     U32* const hashTable  = zc->hashTable;
     const U32 hashLog = zc->params.hashLog;
@@ -210,6 +210,7 @@ static void ZSTD_HC_insert (ZSTD_HC_CCtx* zc, const BYTE* ip, U32 mls)
     }
 
     zc->nextToUpdate = target;
+	return hashTable[ZSTD_HC_hashPtr(ip, hashLog, mls)];
 }
 
 
@@ -220,8 +221,6 @@ size_t ZSTD_HC_insertAndFindBestMatch (
                         const BYTE** matchpos,
                         const U32 maxNbAttempts, const U32 matchLengthSearch)
 {
-    U32* const hashTable = zc->hashTable;
-    const U32 hashLog = zc->params.hashLog;
     U32* const chainTable = zc->chainTable;
     const U32 chainSize = (1 << zc->params.chainLog);
     const U32 chainMask = chainSize-1;
@@ -236,8 +235,7 @@ size_t ZSTD_HC_insertAndFindBestMatch (
     size_t ml=0;
 
     /* HC4 match finder */
-    ZSTD_HC_insert(zc, ip, matchLengthSearch);
-    matchIndex = hashTable[ZSTD_HC_hashPtr(ip, hashLog, matchLengthSearch)];
+    matchIndex = ZSTD_HC_insertAndFindFirstIndex (zc, ip, matchLengthSearch);
 
     while ((matchIndex>=lowLimit) && (nbAttempts))
     {
@@ -275,7 +273,7 @@ size_t ZSTD_HC_insertAndFindBestMatch (
 }
 
 
-static size_t ZSTD_HC_insertAndFindBestMatch_selectMLS (
+FORCE_INLINE size_t ZSTD_HC_insertAndFindBestMatch_selectMLS (
                         ZSTD_HC_CCtx* zc,   /* Index table will be updated */
                         const BYTE* ip, const BYTE* const iLimit,
                         const BYTE** matchpos,
@@ -291,7 +289,108 @@ static size_t ZSTD_HC_insertAndFindBestMatch_selectMLS (
 }
 
 
-static size_t ZSTD_HC_compressBlock(ZSTD_HC_CCtx* ctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize)
+size_t ZSTD_HC_compressBlock_lazy(ZSTD_HC_CCtx* ctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize)
+{
+    seqStore_t* seqStorePtr = &(ctx->seqStore);
+    const BYTE* const istart = (const BYTE*)src;
+    const BYTE* ip = istart;
+    const BYTE* anchor = istart;
+    const BYTE* const iend = istart + srcSize;
+    const BYTE* const ilimit = iend - 8;
+    const BYTE* match = istart;
+
+    size_t offset_2=REPCODE_STARTVALUE, offset_1=REPCODE_STARTVALUE;
+    const U32 maxSearches = 1 << ctx->params.searchLog;
+    const U32 mls = ctx->params.searchLength;
+
+    /* init */
+    ZSTD_resetSeqStore(seqStorePtr);
+    if (((ip-ctx->base) - ctx->dictLimit) < REPCODE_STARTVALUE) ip += REPCODE_STARTVALUE;
+
+    /* Match Loop */
+    while (ip <= ilimit)
+    {
+        size_t matchLength;
+        size_t offset;
+        const BYTE* start;
+
+        /* try to find a first match */
+        if (MEM_read32(ip) == MEM_read32(ip - offset_2))
+        {
+            /* repcode : we take it*/
+            size_t offtmp = offset_2;
+            size_t litLength = ip - anchor;
+            matchLength = ZSTD_count(ip+MINMATCH, ip+MINMATCH-offset_2, iend);
+            offset_2 = offset_1;
+            offset_1 = offtmp;
+            ZSTD_storeSeq(seqStorePtr, litLength, anchor, 0, matchLength);
+            ip += matchLength+MINMATCH;
+            anchor = ip;
+            continue;
+        }
+
+        offset_2 = offset_1;
+        matchLength = ZSTD_HC_insertAndFindBestMatch_selectMLS(ctx, ip, iend, &match, maxSearches, mls);
+        if (!matchLength) { ip++; continue; }
+
+        /* let's try to find a better solution */
+        offset = ip - match;
+        start = ip;
+
+        while (ip<ilimit)
+        {
+            ip ++;
+            if (MEM_read32(ip) == MEM_read32(ip - offset_1))
+            {
+                size_t ml2 = ZSTD_count(ip+MINMATCH, ip+MINMATCH-offset_1, iend) + MINMATCH;
+                int gain2 = (int)(ml2 * 4);
+                int gain1 = (int)(matchLength*4 - ZSTD_highbit((U32)offset));
+                if (gain2 > gain1)
+                {
+                    matchLength = ml2, offset = 0, start = ip;
+                    break;
+                }
+            }
+            {
+                size_t ml2 = ZSTD_HC_insertAndFindBestMatch_selectMLS(ctx, ip, iend, &match, maxSearches, mls);
+                size_t offset2 = ip - match;
+                int gain2 = (int)(ml2*5 - ZSTD_highbit((U32)offset2));   /* raw approx */
+                int gain1 = (int)(matchLength*5 - ZSTD_highbit((U32)offset));
+                if (gain2 > gain1)
+                {
+                    matchLength = ml2, offset = offset2, start = ip;
+                    continue;   /* search a better one */
+                }
+            }
+
+            break;  /* nothing found : store previous one */
+        }
+
+        /* store sequence */
+        {
+            size_t litLength = start - anchor;
+            if (offset) offset_1 = offset;
+            ZSTD_storeSeq(seqStorePtr, litLength, anchor, offset, matchLength-MINMATCH);
+            ip = start + matchLength;
+            anchor = ip;
+        }
+
+    }
+
+    /* Last Literals */
+    {
+        size_t lastLLSize = iend - anchor;
+        memcpy(seqStorePtr->lit, anchor, lastLLSize);
+        seqStorePtr->lit += lastLLSize;
+    }
+
+    /* Final compression stage */
+    return ZSTD_compressSequences((BYTE*)dst, maxDstSize,
+                                  seqStorePtr, srcSize);
+}
+
+
+size_t ZSTD_HC_compressBlock_greedy(ZSTD_HC_CCtx* ctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize)
 {
     seqStore_t* seqStorePtr = &(ctx->seqStore);
     const BYTE* const istart = (const BYTE*)src;
@@ -364,9 +463,19 @@ static size_t ZSTD_HC_compressBlock(ZSTD_HC_CCtx* ctx, void* dst, size_t maxDstS
 
     /* Final compression stage */
     return ZSTD_compressSequences((BYTE*)dst, maxDstSize,
+
                                   seqStorePtr, srcSize);
 }
 
+
+size_t ZSTD_HC_compressBlock(ZSTD_HC_CCtx* ctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize)
+{
+	if (ctx->params.strategy == ZSTD_HC_greedy)
+	return ZSTD_HC_compressBlock_greedy(ctx, dst, maxDstSize, src, srcSize);
+	return ZSTD_HC_compressBlock_lazy(ctx, dst, maxDstSize, src, srcSize);
+}
+
+
 static size_t ZSTD_HC_compress_generic (ZSTD_HC_CCtx* ctxPtr,
                                         void* dst, size_t maxDstSize,
                                   const void* src, size_t srcSize)
@@ -377,10 +486,17 @@ static size_t ZSTD_HC_compress_generic (ZSTD_HC_CCtx* ctxPtr,
     BYTE* const ostart = (BYTE*)dst;
     BYTE* op = ostart;
     BYTE* const oend = op + maxDstSize;
+    size_t (*blockCompressor) (ZSTD_HC_CCtx* ctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize);
+
+    if (ctxPtr->params.strategy == ZSTD_HC_greedy)
+		blockCompressor = ZSTD_HC_compressBlock_greedy;
+	else
+		blockCompressor = ZSTD_HC_compressBlock_lazy;
+
 
     while (remaining > blockSize)
     {
-        size_t cSize = ZSTD_HC_compressBlock(ctxPtr, op+3, oend-op, ip, blockSize);
+        size_t cSize = blockCompressor(ctxPtr, op+3, oend-op, ip, blockSize);
 
         if (cSize == 0)
         {
@@ -403,7 +519,7 @@ static size_t ZSTD_HC_compress_generic (ZSTD_HC_CCtx* ctxPtr,
 
     /* last block */
     {
-        size_t cSize = ZSTD_HC_compressBlock(ctxPtr, op+3, oend-op, ip, remaining);
+        size_t cSize = blockCompressor(ctxPtr, op+3, oend-op, ip, remaining);
 
         if (cSize == 0)
         {

lib/zstdhc_static.h

@@ -45,6 +45,7 @@ extern "C" {
 /* *************************************
 *  Types
 ***************************************/
+typedef enum { ZSTD_HC_greedy, ZSTD_HC_lazy } ZSTD_HC_strategy;
 typedef struct
 {
     U32 windowLog;     /* largest match distance : impact decompression buffer size */
@@ -52,6 +53,7 @@ typedef struct
     U32 hashLog;       /* dispatch table : larger == more memory, faster*/
     U32 searchLog;     /* nb of searches : larger == more compression, slower*/
     U32 searchLength;  /* size of matches : larger == faster decompression */
+    ZSTD_HC_strategy strategy;   /* greedy, lazy (stronger, slower) */
 } ZSTD_HC_parameters;
 
 /* parameters boundaries */
@@ -91,34 +93,34 @@ size_t ZSTD_HC_compressEnd(ZSTD_HC_CCtx* ctx, void* dst, size_t maxDstSize);
 ***************************************/
 #define ZSTD_HC_MAX_CLEVEL 26
 static const ZSTD_HC_parameters ZSTD_HC_defaultParameters[ZSTD_HC_MAX_CLEVEL+1] = {
-    /* W,  C,  H,  S */
-    { 18, 12, 14,  1,  4 },   /* level  0 - never used */
-    { 18, 12, 14,  1,  4 },   /* level  1 - in fact redirected towards zstd fast */
-    { 18, 12, 15,  2,  4 },   /* level  2 */
-    { 19, 14, 16,  3,  4 },   /* level  3 */
-    { 20, 15, 17,  4,  5 },   /* level  4 */
-    { 20, 17, 19,  4,  5 },   /* level  5 */
-    { 20, 19, 19,  4,  5 },   /* level  6 */
-    { 20, 19, 19,  5,  5 },   /* level  7 */
-    { 20, 20, 20,  5,  5 },   /* level  8 */
-    { 20, 20, 20,  6,  5 },   /* level  9 */
-    { 21, 21, 20,  5,  5 },   /* level 10 */
-    { 22, 21, 22,  6,  5 },   /* level 11 */
-    { 23, 21, 22,  6,  5 },   /* level 12 */
-    { 23, 21, 22,  7,  5 },   /* level 13 */
-    { 22, 22, 23,  7,  5 },   /* level 14 */
-    { 22, 22, 23,  7,  5 },   /* level 15 */
-    { 22, 22, 23,  8,  5 },   /* level 16 */
-    { 22, 22, 23,  8,  5 },   /* level 17 */
-    { 22, 22, 23,  9,  5 },   /* level 18 */
-    { 22, 22, 23,  9,  5 },   /* level 19 */
-    { 23, 23, 23,  9,  5 },   /* level 20 */
-    { 23, 23, 23,  9,  5 },   /* level 21 */
-    { 23, 23, 23, 10,  5 },   /* level 22 */
-    { 23, 23, 23, 10,  5 },   /* level 23 */
-    { 23, 23, 23, 11,  5 },   /* level 24 */
-    { 23, 23, 23, 12,  5 },   /* level 25 */
-    { 23, 23, 23, 13,  5 },   /* level 26 */   /* ZSTD_HC_MAX_CLEVEL */
+    /* W,  C,  H,  S,  L, strat */
+    { 18, 12, 14,  1,  4, ZSTD_HC_greedy },   /* level  0 - never used */
+    { 18, 12, 14,  1,  4, ZSTD_HC_greedy },   /* level  1 - in fact redirected towards zstd fast */
+    { 18, 12, 15,  2,  4, ZSTD_HC_greedy },   /* level  2 */
+    { 19, 13, 17,  3,  5, ZSTD_HC_greedy },   /* level  3 */
+    { 20, 18, 19,  2,  5, ZSTD_HC_greedy },   /* level  4 */
+    { 20, 19, 19,  3,  5, ZSTD_HC_greedy },   /* level  5 */
+    { 20, 18, 20,  3,  5, ZSTD_HC_lazy   },   /* level  6 */
+    { 20, 18, 20,  4,  5, ZSTD_HC_lazy   },   /* level  7 */
+    { 21, 19, 20,  4,  5, ZSTD_HC_lazy   },   /* level  8 */
+    { 21, 19, 20,  5,  5, ZSTD_HC_lazy   },   /* level  9 */
+    { 21, 20, 20,  5,  5, ZSTD_HC_lazy   },   /* level 10 */
+    { 21, 21, 20,  5,  5, ZSTD_HC_lazy   },   /* level 11 */
+    { 22, 20, 22,  6,  5, ZSTD_HC_lazy   },   /* level 12 */
+    { 22, 21, 22,  6,  5, ZSTD_HC_lazy   },   /* level 13 */
+    { 23, 21, 22,  6,  5, ZSTD_HC_lazy   },   /* level 14 */
+    { 23, 21, 23,  7,  5, ZSTD_HC_lazy   },   /* level 15 */
+    { 23, 22, 22,  6,  5, ZSTD_HC_lazy   },   /* level 16 */
+    { 23, 22, 22,  7,  5, ZSTD_HC_lazy   },   /* level 17 */
+    { 23, 23, 22,  7,  5, ZSTD_HC_lazy   },   /* level 18 */
+    { 23, 22, 23,  8,  5, ZSTD_HC_lazy   },   /* level 19 */
+    { 23, 23, 23,  8,  5, ZSTD_HC_lazy   },   /* level 20 */
+    { 23, 23, 23,  8,  5, ZSTD_HC_lazy   },   /* level 21 */
+    { 24, 24, 24,  8,  5, ZSTD_HC_lazy   },   /* level 22 */
+    { 24, 23, 23,  9,  5, ZSTD_HC_lazy   },   /* level 23 */
+    { 24, 24, 24,  9,  5, ZSTD_HC_lazy   },   /* level 24 */
+    { 24, 24, 24,  9,  5, ZSTD_HC_lazy   },   /* level 25 */
+    { 24, 24, 24, 10,  5, ZSTD_HC_lazy   },   /* level 26 */     /* ZSTD_HC_MAX_CLEVEL */
 };
 
 #if defined (__cplusplus)

programs/Makefile

@@ -30,7 +30,7 @@
 # fullbench32: Same as fullbench, but forced to compile in 32-bits mode
 # ##########################################################################
 
-VERSION?= 0.3.0
+VERSION?= 0.3.2
 
 DESTDIR?=
 PREFIX ?= /usr/local

programs/bench.c

@@ -237,7 +237,7 @@ static size_t local_compress_fast (void* dst, size_t maxDstSize, const void* src
     return ZSTD_compress(dst, maxDstSize, src, srcSize);
 }
 
-#define MIN(a,b) (a<b ? a : b)
+#define MIN(a,b) ((a)<(b) ? (a) : (b))
 
 static int BMK_benchMem(void* srcBuffer, size_t srcSize, const char* fileName, int cLevel)
 {

programs/paramgrill.c

@@ -122,7 +122,8 @@ static U32 g_rand = 1;
 static U32 g_singleRun = 0;
 static U32 g_target = 0;
 static U32 g_noSeed = 0;
-static ZSTD_HC_parameters g_params = { 0, 0, 0, 0, 0 };
+static const ZSTD_HC_parameters* g_seedParams = ZSTD_HC_defaultParameters;
+static ZSTD_HC_parameters g_params = { 0, 0, 0, 0, 0, ZSTD_HC_greedy };
 
 void BMK_SetNbIterations(int nbLoops)
 {
@@ -283,9 +284,12 @@ static size_t BMK_benchParam(BMK_result_t* resultPtr,
     U32 Hlog = params.hashLog;
     U32 Slog = params.searchLog;
     U32 Slength = params.searchLength;
+    ZSTD_HC_strategy strat = params.strategy;
+    char name[30] = { 0 };
     U64 crcOrig;
 
     /* Memory allocation & restrictions */
+    snprintf(name, 30, "W%02uC%02uH%02uS%02uL%1ust%1u", Wlog, Clog, Hlog, Slog, Slength, strat);
     if (!compressedBuffer || !resultBuffer || !blockTable)
     {
         DISPLAY("\nError: not enough memory!\n");
@@ -344,7 +348,7 @@ static size_t BMK_benchParam(BMK_result_t* resultPtr,
             if (totalTime > g_maxParamTime) break;
 
             /* Compression */
-            DISPLAY("%1u-W%02uC%02uH%02uS%02uL%02u : %9u ->\r", loopNb, Wlog, Clog, Hlog, Slog, Slength, (U32)srcSize);
+            DISPLAY("%1u-%s : %9u ->\r", loopNb, name, (U32)srcSize);
             memset(compressedBuffer, 0xE5, maxCompressedSize);
 
             nbLoops = 0;
@@ -367,7 +371,7 @@ static size_t BMK_benchParam(BMK_result_t* resultPtr,
                 cSize += blockTable[blockNb].cSize;
             if ((double)milliTime < fastestC*nbLoops) fastestC = (double)milliTime / nbLoops;
             ratio = (double)srcSize / (double)cSize;
-            DISPLAY("%1u-W%02uC%02uH%02uS%02uL%02u : %9u ->", loopNb, Wlog, Clog, Hlog, Slog, Slength, (U32)srcSize);
+            DISPLAY("%1u-%s : %9u ->", loopNb, name, (U32)srcSize);
             DISPLAY(" %9u (%4.3f),%7.1f MB/s\r", (U32)cSize, ratio, (double)srcSize / fastestC / 1000.);
             resultPtr->cSize = cSize;
             resultPtr->cSpeed = (U32)((double)srcSize / fastestC);
@@ -389,7 +393,7 @@ static size_t BMK_benchParam(BMK_result_t* resultPtr,
             milliTime = BMK_GetMilliSpan(milliTime);
 
             if ((double)milliTime < fastestD*nbLoops) fastestD = (double)milliTime / nbLoops;
-            DISPLAY("%1u-W%02uC%02uH%02uS%02uL%02u : %9u -> ", loopNb, Wlog, Clog, Hlog, Slog, Slength, (U32)srcSize);
+            DISPLAY("%1u-%s : %9u -> ", loopNb, name, (U32)srcSize);
             DISPLAY("%9u (%4.3f),%7.1f MB/s, ", (U32)cSize, ratio, (double)srcSize / fastestC / 1000.);
             DISPLAY("%7.1f MB/s\r", (double)srcSize / fastestD / 1000.);
             resultPtr->dSpeed = (U32)((double)srcSize / fastestD);
@@ -421,11 +425,14 @@ static size_t BMK_benchParam(BMK_result_t* resultPtr,
     return 0;
 }
 
+const char* g_stratName[2] = { "ZSTD_HC_greedy", "ZSTD_HC_lazy  " };
 
 static void BMK_printWinner(FILE* f, U32 cLevel, BMK_result_t result, ZSTD_HC_parameters params, size_t srcSize)
 {
     DISPLAY("\r%79s\r", "");
-    fprintf(f,"    {%3u,%3u,%3u,%3u,%3u },   ", params.windowLog, params.chainLog, params.hashLog, params.searchLog, params.searchLength);
+    fprintf(f,"    {%3u,%3u,%3u,%3u,%3u, %s },   ",
+            params.windowLog, params.chainLog, params.hashLog, params.searchLog, params.searchLength,
+            g_stratName[params.strategy]);
     fprintf(f,
             "/* level %2u */     /* R:%5.3f at %5.1f MB/s - %5.1f MB/s */ \n",
             cLevel, (double)srcSize / result.cSize, (double)result.cSpeed / 1000., (double)result.dSpeed / 1000.);
@@ -446,7 +453,7 @@ static void BMK_printWinners2(FILE* f, const winnerInfo_t* winners, size_t srcSi
     fprintf(f, "\n /* Selected configurations : */ \n");
     fprintf(f, "#define ZSTD_HC_MAX_CLEVEL %2u \n", ZSTD_HC_MAX_CLEVEL);
     fprintf(f, "static const ZSTD_HC_parameters ZSTD_HC_defaultParameters[ZSTD_HC_MAX_CLEVEL+1] = {\n");
-    fprintf(f, "    /* W,  C,  H,  S,  L */ \n");
+    fprintf(f, "    /* W,  C,  H,  S,  L, strat */ \n");
 
     for (cLevel=0; cLevel <= ZSTD_HC_MAX_CLEVEL; cLevel++)
         BMK_printWinner(f, cLevel, winners[cLevel].result, winners[cLevel].params, srcSize);
@@ -486,7 +493,7 @@ static int BMK_seed(winnerInfo_t* winners, const ZSTD_HC_parameters params,
             continue;
         }
 
-        if ((double)testResult.cSize <= ((double)winners[cLevel].result.cSize * (1. + (0.01 / cLevel))) )
+        if ((double)testResult.cSize <= ((double)winners[cLevel].result.cSize * (1. + (0.02 / cLevel))) )
         {
             /* Validate solution is "good enough" */
             double W_ratio = (double)srcSize / testResult.cSize;
@@ -567,13 +574,17 @@ static int BMK_seed(winnerInfo_t* winners, const ZSTD_HC_parameters params,
 static BYTE g_alreadyTested[ZSTD_HC_WINDOWLOG_MAX+1-ZSTD_HC_WINDOWLOG_MIN]
                            [ZSTD_HC_CHAINLOG_MAX+1-ZSTD_HC_CHAINLOG_MIN]
                            [ZSTD_HC_HASHLOG_MAX+1-ZSTD_HC_HASHLOG_MIN]
-                           [ZSTD_HC_SEARCHLOG_MAX+1-ZSTD_HC_SEARCHLOG_MIN] = {};   /* init to zero */
+                           [ZSTD_HC_SEARCHLOG_MAX+1-ZSTD_HC_SEARCHLOG_MIN]
+                           [ZSTD_HC_SEARCHLENGTH_MAX+1-ZSTD_HC_SEARCHLENGTH_MIN]
+                           [2 /* strategy */ ] = {};   /* init to zero */
 
 #define NB_TESTS_PLAYED(p) \
     g_alreadyTested[p.windowLog-ZSTD_HC_WINDOWLOG_MIN] \
                    [p.chainLog-ZSTD_HC_CHAINLOG_MIN]   \
                    [p.hashLog-ZSTD_HC_HASHLOG_MIN]     \
-                   [p.searchLog-ZSTD_HC_SEARCHLOG_MIN]
+                   [p.searchLog-ZSTD_HC_SEARCHLOG_MIN] \
+                   [p.searchLength-ZSTD_HC_SEARCHLENGTH_MIN] \
+                   [(U32)p.strategy]
 
 
 static void playAround(FILE* f, winnerInfo_t* winners,
@@ -593,7 +604,7 @@ static void playAround(FILE* f, winnerInfo_t* winners,
 
         for (; nbChanges; nbChanges--)
         {
-            const U32 changeID = FUZ_rand(&g_rand) % 9;
+            const U32 changeID = FUZ_rand(&g_rand) % 12;
             switch(changeID)
             {
             case 0:
@@ -616,6 +627,10 @@ static void playAround(FILE* f, winnerInfo_t* winners,
                 p.searchLength++; break;
             case 9:
                 p.searchLength--; break;
+            case 10:
+                p.strategy = ZSTD_HC_lazy; break;
+            case 11:
+                p.strategy = ZSTD_HC_greedy; break;
             }
         }
 
@@ -631,9 +646,12 @@ static void playAround(FILE* f, winnerInfo_t* winners,
         if (p.searchLog < ZSTD_HC_SEARCHLOG_MIN) continue;
         if (p.searchLength > ZSTD_HC_SEARCHLENGTH_MAX) continue;
         if (p.searchLength < ZSTD_HC_SEARCHLENGTH_MIN) continue;
+        if (p.strategy < ZSTD_HC_greedy) continue;
+        if (p.strategy > ZSTD_HC_lazy) continue;
 
         /* exclude faster if already played params */
-        if (FUZ_rand(&g_rand) & ((1 << NB_TESTS_PLAYED(p))-1)) continue;
+        if (FUZ_rand(&g_rand) & ((1 << NB_TESTS_PLAYED(p))-1))
+            continue;
 
         /* test */
         NB_TESTS_PLAYED(p)++;
@@ -662,6 +680,7 @@ static void BMK_selectRandomStart(
         p.searchLog  = FUZ_rand(&g_rand) % (ZSTD_HC_SEARCHLOG_MAX+1 - ZSTD_HC_SEARCHLOG_MIN) + ZSTD_HC_SEARCHLOG_MIN;
         p.windowLog  = FUZ_rand(&g_rand) % (ZSTD_HC_WINDOWLOG_MAX+1 - ZSTD_HC_WINDOWLOG_MIN) + ZSTD_HC_WINDOWLOG_MIN;
         p.searchLength=FUZ_rand(&g_rand) % (ZSTD_HC_SEARCHLENGTH_MAX+1 - ZSTD_HC_SEARCHLENGTH_MIN) + ZSTD_HC_SEARCHLENGTH_MIN;
+        p.strategy   = (ZSTD_HC_strategy) (FUZ_rand(&g_rand) & 1);
         playAround(f, winners, p, srcBuffer, srcSize, ctx);
     }
     else
@@ -669,8 +688,6 @@ static void BMK_selectRandomStart(
 }
 
 
-static const ZSTD_HC_parameters* g_seedParams = ZSTD_HC_defaultParameters;
-
 static void BMK_benchMem(void* srcBuffer, size_t srcSize)
 {
     ZSTD_HC_CCtx* ctx = ZSTD_HC_createCCtx();
@@ -955,12 +972,20 @@ int main(int argc, char** argv)
                             while ((*argument>= '0') && (*argument<='9'))
                                 g_params.searchLog *= 10, g_params.searchLog += *argument++ - '0';
                             continue;
-                        case 'l':
+                        case 'l':  /* search length */
                             g_params.searchLength = 0;
                             argument++;
                             while ((*argument>= '0') && (*argument<='9'))
                                 g_params.searchLength *= 10, g_params.searchLength += *argument++ - '0';
                             continue;
+                        case 't':  /* strategy */
+                            g_params.strategy = ZSTD_HC_greedy;
+                            argument++;
+                            while ((*argument>= '0') && (*argument<='9'))
+                            {
+                                if (*argument++) g_params.strategy = ZSTD_HC_lazy;
+                            }
+                            continue;
                         case 'L':
                             {
                                 int cLevel = 0;

programs/zstdcli.c

@@ -70,7 +70,7 @@
 **************************************/
 #define COMPRESSOR_NAME "zstd command line interface"
 #ifndef ZSTD_VERSION
-#  define ZSTD_VERSION "v0.0.1"
+#  define ZSTD_VERSION "v0.3.0"
 #endif
 #define AUTHOR "Yann Collet"
 #define WELCOME_MESSAGE "*** %s %i-bits %s, by %s (%s) ***\n", COMPRESSOR_NAME, (int)(sizeof(void*)*8), ZSTD_VERSION, AUTHOR, __DATE__

visual/2012/fullbench/fullbench.vcxproj

@@ -78,7 +78,7 @@
   </PropertyGroup>
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
     <LinkIncremental>false</LinkIncremental>
-    <IncludePath>$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
+    <IncludePath>$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
     <RunCodeAnalysis>true</RunCodeAnalysis>
   </PropertyGroup>
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">

visual/2012/fuzzer/fuzzer.vcxproj

@@ -78,7 +78,7 @@
   </PropertyGroup>
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
     <LinkIncremental>false</LinkIncremental>
-    <IncludePath>$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
+    <IncludePath>$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
     <RunCodeAnalysis>true</RunCodeAnalysis>
   </PropertyGroup>
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">

visual/2012/zstd/zstd.vcxproj

@@ -109,7 +109,7 @@
   </PropertyGroup>
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
     <LinkIncremental>false</LinkIncremental>
-    <IncludePath>$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
+    <IncludePath>$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
     <RunCodeAnalysis>true</RunCodeAnalysis>
   </PropertyGroup>
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">