small blocks params

Makefile

@@ -32,7 +32,7 @@
 # ################################################################
 
 # Version number
-export VERSION := 0.3.5
+export VERSION := 0.3.6
 
 PRGDIR  = programs
 ZSTDDIR = lib

NEWS

@@ -1,3 +1,6 @@
+v0.3.6
+small blocks params
+
 v0.3.5
 minor generic compression improvements
 

lib/zstd.h

@@ -48,7 +48,7 @@ extern "C" {
 ***************************************/
 #define ZSTD_VERSION_MAJOR    0    /* for breaking interface changes  */
 #define ZSTD_VERSION_MINOR    3    /* for new (non-breaking) interface capabilities */
-#define ZSTD_VERSION_RELEASE  5    /* for tweaks, bug-fixes, or development */
+#define ZSTD_VERSION_RELEASE  6    /* for tweaks, bug-fixes, or development */
 #define ZSTD_VERSION_NUMBER  (ZSTD_VERSION_MAJOR *100*100 + ZSTD_VERSION_MINOR *100 + ZSTD_VERSION_RELEASE)
 unsigned ZSTD_versionNumber (void);
 

lib/zstd_internal.h

@@ -226,6 +226,14 @@ static const U32 g_searchStrength = 8;
 */
 MEM_STATIC void ZSTD_storeSeq(seqStore_t* seqStorePtr, size_t litLength, const BYTE* literals, size_t offsetCode, size_t matchCode)
 {
+#if 0
+    static const BYTE* g_start = NULL;
+    if (g_start==NULL) g_start = literals;
+    if (literals - g_start == 8695)
+    printf("pos %6u : %3u literals & match %3u bytes at distance %6u \n",
+           (U32)(literals - g_start), (U32)litLength, (U32)matchCode+4, (U32)offsetCode);
+#endif
+
     /* copy Literals */
     ZSTD_wildcopy(seqStorePtr->lit, literals, litLength);
     seqStorePtr->lit += litLength;

lib/zstdhc.c

@@ -954,9 +954,10 @@ size_t ZSTD_HC_compressBegin_advanced(ZSTD_HC_CCtx* ctx,
 
 size_t ZSTD_HC_compressBegin(ZSTD_HC_CCtx* ctx, void* dst, size_t maxDstSize, int compressionLevel)
 {
+    int tableID = 1;
     if (compressionLevel<=0) compressionLevel = 1;
     if (compressionLevel > ZSTD_HC_MAX_CLEVEL) compressionLevel = ZSTD_HC_MAX_CLEVEL;
-    return ZSTD_HC_compressBegin_advanced(ctx, dst, maxDstSize, ZSTD_HC_defaultParameters[compressionLevel]);
+    return ZSTD_HC_compressBegin_advanced(ctx, dst, maxDstSize, ZSTD_HC_defaultParameters[tableID][compressionLevel]);
 }
 
 
@@ -1016,9 +1017,10 @@ size_t ZSTD_HC_compress_advanced (ZSTD_HC_CCtx* ctx,
 
 size_t ZSTD_HC_compressCCtx (ZSTD_HC_CCtx* ctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize, int compressionLevel)
 {
+    const int tableID = (srcSize > 128 KB);
     if (compressionLevel<=1) return ZSTD_compress(dst, maxDstSize, src, srcSize);   /* fast mode */
     if (compressionLevel > ZSTD_HC_MAX_CLEVEL) compressionLevel = ZSTD_HC_MAX_CLEVEL;
-    return ZSTD_HC_compress_advanced(ctx, dst, maxDstSize, src, srcSize, ZSTD_HC_defaultParameters[compressionLevel]);
+    return ZSTD_HC_compress_advanced(ctx, dst, maxDstSize, src, srcSize, ZSTD_HC_defaultParameters[tableID][compressionLevel]);
 }
 
 size_t ZSTD_HC_compress(void* dst, size_t maxDstSize, const void* src, size_t srcSize, int compressionLevel)

lib/zstdhc_static.h

@@ -99,7 +99,32 @@ size_t ZSTD_HC_compressEnd(ZSTD_HC_CCtx* ctx, void* dst, size_t maxDstSize);
 *  Pre-defined compression levels
 ***************************************/
 #define ZSTD_HC_MAX_CLEVEL 20
-static const ZSTD_HC_parameters ZSTD_HC_defaultParameters[ZSTD_HC_MAX_CLEVEL+1] = {
+static const ZSTD_HC_parameters ZSTD_HC_defaultParameters[2][ZSTD_HC_MAX_CLEVEL+1] = {
+{   /* for <= 128 KB */
+    /* W,  C,  H,  S,  L, strat */
+    { 17, 12, 12,  1,  4, ZSTD_HC_fast    },  /* level  0 - never used */
+    { 17, 12, 13,  1,  6, ZSTD_HC_fast    },  /* level  1 */
+    { 17, 15, 16,  1,  5, ZSTD_HC_fast    },  /* level  2 */
+    { 17, 16, 17,  1,  5, ZSTD_HC_fast    },  /* level  3 */
+    { 17, 13, 15,  2,  4, ZSTD_HC_greedy  },  /* level  4 */
+    { 17, 15, 17,  3,  4, ZSTD_HC_greedy  },  /* level  5 */
+    { 17, 14, 17,  3,  4, ZSTD_HC_lazy    },  /* level  6 */
+    { 17, 16, 17,  4,  4, ZSTD_HC_lazy    },  /* level  7 */
+    { 17, 16, 17,  4,  4, ZSTD_HC_lazy2   },  /* level  8 */
+    { 17, 17, 16,  5,  4, ZSTD_HC_lazy2   },  /* level  9 */
+    { 17, 17, 16,  6,  4, ZSTD_HC_lazy2   },  /* level 10 */
+    { 17, 17, 16,  7,  4, ZSTD_HC_lazy2   },  /* level 11 */
+    { 17, 17, 16,  8,  4, ZSTD_HC_lazy2   },  /* level 12 */
+    { 17, 17, 16,  9,  4, ZSTD_HC_lazy2   },  /* level 13 */
+    { 17, 18, 16,  5,  4, ZSTD_HC_btlazy2 },  /* level 14 */
+    { 17, 18, 16,  6,  4, ZSTD_HC_btlazy2 },  /* level 15 */
+    { 17, 18, 16,  7,  4, ZSTD_HC_btlazy2 },  /* level 16 */
+    { 17, 18, 16,  8,  4, ZSTD_HC_btlazy2 },  /* level 17 */
+    { 17, 18, 16,  9,  4, ZSTD_HC_btlazy2 },  /* level 18 */
+    { 17, 18, 16, 10,  4, ZSTD_HC_btlazy2 },  /* level 19 */
+    { 17, 18, 18, 12,  4, ZSTD_HC_btlazy2 },   /* level 20 */
+},
+{   /* for > 128 KB */
     /* W,  C,  H,  S,  L, strat */
     { 18, 12, 12,  1,  4, ZSTD_HC_fast    },  /* level  0 - never used */
     { 18, 14, 14,  1,  7, ZSTD_HC_fast    },  /* level  1 - in fact redirected towards zstd fast */
@@ -122,6 +147,7 @@ static const ZSTD_HC_parameters ZSTD_HC_defaultParameters[ZSTD_HC_MAX_CLEVEL+1]
     { 25, 24, 23,  5,  5, ZSTD_HC_btlazy2 },  /* level 18 */
     { 25, 26, 23,  5,  5, ZSTD_HC_btlazy2 },  /* level 19 */
     { 26, 27, 24,  6,  5, ZSTD_HC_btlazy2 },  /* level 20 */
+}
 };
 
 

programs/Makefile

@@ -30,7 +30,7 @@
 # fullbench32: Same as fullbench, but forced to compile in 32-bits mode
 # ##########################################################################
 
-VERSION?= 0.3.5
+VERSION?= 0.3.6
 
 DESTDIR?=
 PREFIX ?= /usr/local

programs/fuzzer.c

@@ -355,7 +355,7 @@ int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibilit
         dSize = ZSTD_decompress(dstBuffer, sampleSize + dSupSize, cBuffer, cSize);
         CHECK(dSize != sampleSize, "ZSTD_decompress failed (%s) (srcSize : %u ; cSize : %u)", ZSTD_getErrorName(dSize), (U32)sampleSize, (U32)cSize);
         crcDest = XXH64(dstBuffer, sampleSize, 0);
-        CHECK(crcOrig != crcDest, "dstBuffer corrupted (pos %u / %u)", (U32)findDiff(srcBuffer+sampleStart, dstBuffer, sampleSize), (U32)sampleSize);
+        CHECK(crcOrig != crcDest, "decompression result corrupted (pos %u / %u)", (U32)findDiff(srcBuffer+sampleStart, dstBuffer, sampleSize), (U32)sampleSize);
 
         /* truncated src decompression test */
         {

programs/paramgrill.c

@@ -122,7 +122,7 @@ static U32 g_rand = 1;
 static U32 g_singleRun = 0;
 static U32 g_target = 0;
 static U32 g_noSeed = 0;
-static const ZSTD_HC_parameters* g_seedParams = ZSTD_HC_defaultParameters;
+static const ZSTD_HC_parameters* g_seedParams = ZSTD_HC_defaultParameters[0];
 static ZSTD_HC_parameters g_params = { 0, 0, 0, 0, 0, ZSTD_HC_greedy };
 
 void BMK_SetNbIterations(int nbLoops)
@@ -703,12 +703,13 @@ static void BMK_benchMem(void* srcBuffer, size_t srcSize)
     int i;
     const char* rfName = "grillResults.txt";
     FILE* f;
-    const U32 srcLog = BMK_highbit((U32)( (g_blockSize ? g_blockSize : srcSize) -1))+1;
+    const size_t blockSize = g_blockSize ? g_blockSize : srcSize;
+    const U32 srcLog = BMK_highbit((U32)(blockSize-1))+1;
 
     if (g_singleRun)
     {
         BMK_result_t testResult;
-        ZSTD_HC_validateParams(&g_params, g_blockSize ? g_blockSize : srcSize);
+        ZSTD_HC_validateParams(&g_params, blockSize);
         BMK_benchParam(&testResult, srcBuffer, srcSize, ctx, g_params);
         DISPLAY("\n");
         return;
@@ -725,12 +726,13 @@ static void BMK_benchMem(void* srcBuffer, size_t srcSize)
     {
         /* baseline config for level 1 */
         BMK_result_t testResult;
-        params.windowLog = MIN(srcLog, 18);
+        params.windowLog = 18;
         params.hashLog = 14;
         params.contentLog = 1;
         params.searchLog = 1;
         params.searchLength = 7;
         params.strategy = ZSTD_HC_fast;
+        ZSTD_HC_validateParams(&params, blockSize);
         BMK_benchParam(&testResult, srcBuffer, srcSize, ctx, params);
         g_cSpeedTarget[1] = (testResult.cSpeed * 15) >> 4;
     }
@@ -741,7 +743,9 @@ static void BMK_benchMem(void* srcBuffer, size_t srcSize)
 
     /* populate initial solution */
     {
+        const int tableID = (blockSize > 128 KB);
         const int maxSeeds = g_noSeed ? 1 : ZSTD_HC_MAX_CLEVEL;
+        g_seedParams = ZSTD_HC_defaultParameters[tableID];
         for (i=1; i<=maxSeeds; i++)
         {
             const U32 btPlus = (params.strategy == ZSTD_HC_btlazy2);

programs/zstdcli.c

@@ -70,7 +70,7 @@
 **************************************/
 #define COMPRESSOR_NAME "zstd command line interface"
 #ifndef ZSTD_VERSION
-#  define ZSTD_VERSION "v0.3.5"
+#  define ZSTD_VERSION "v0.3.6"
 #endif
 #define AUTHOR "Yann Collet"
 #define WELCOME_MESSAGE "*** %s %i-bits %s, by %s (%s) ***\n", COMPRESSOR_NAME, (int)(sizeof(void*)*8), ZSTD_VERSION, AUTHOR, __DATE__