statistics of encoded sequences

lib/zstd_compress.c

@@ -726,6 +726,12 @@ MEM_STATIC void ZSTD_storeSeq(seqStore_t* seqStorePtr, size_t litLength, const B
     printf("pos %6u : %3u literals & match %3u bytes at distance %6u \n",
            (U32)(literals - g_start), (U32)litLength, (U32)matchCode+MINMATCH, (U32)offsetCode);
 #endif
+#if ZSTD_OPT_DEBUG >= 3
+    if (offsetCode == 0) seqStorePtr->realRepSum++;
+    seqStorePtr->realSeqSum++;
+    seqStorePtr->realMatchSum += matchCode;
+    seqStorePtr->realLitSum += litLength;
+#endif
 
     /* copy Literals */
     ZSTD_wildcopy(seqStorePtr->lit, literals, litLength);
@@ -1912,6 +1918,9 @@ static size_t ZSTD_compress_generic (ZSTD_CCtx* zc,
     BYTE* const ostart = (BYTE*)dst;
     BYTE* op = ostart;
     const U32 maxDist = 1 << zc->params.windowLog;
+    seqStore_t* ssPtr = &zc->seqStore;
+
+    ssPtr->realMatchSum = ssPtr->realLitSum = ssPtr->realSeqSum = ssPtr->realRepSum = 0;
 
     while (remaining) {
         size_t cSize;
@@ -1945,6 +1954,11 @@ static size_t ZSTD_compress_generic (ZSTD_CCtx* zc,
         op += cSize;
     }
 
+#if ZSTD_OPT_DEBUG >= 3
+    ssPtr->realMatchSum += ssPtr->realSeqSum * ((zc->params.searchLength == 3) ? 3 : 4);
+    printf("avgMatchL=%.2f avgLitL=%.2f match=%.1f%% lit=%.1f%% reps=%d seq=%d\n", (float)ssPtr->realMatchSum/ssPtr->realSeqSum, (float)ssPtr->realLitSum/ssPtr->realSeqSum, 100.0*ssPtr->realMatchSum/(ssPtr->realMatchSum+ssPtr->realLitSum), 100.0*ssPtr->realLitSum/(ssPtr->realMatchSum+ssPtr->realLitSum), ssPtr->realRepSum, ssPtr->realSeqSum);
+#endif
+
     return op-ostart;
 }
 

lib/zstd_internal.h

@@ -50,7 +50,7 @@
 /*-*************************************
 *  Common constants
 ***************************************/
-#define ZSTD_OPT_DEBUG 0     // 1 = tableID=0;  5 = check encoded sequences;  9 = full logs
+#define ZSTD_OPT_DEBUG 3     // 1 = tableID=0;  3 = print block stats;  5 = check encoded sequences;  9 = full logs
 #if ZSTD_OPT_DEBUG > 0
     #include <stdio.h>  /* for debug */
 #endif
@@ -179,6 +179,10 @@ typedef struct {
     U32  litLengthSum;
     U32  litSum;
     U32  offCodeSum;
+    U32  realMatchSum;
+    U32  realLitSum;
+    U32  realSeqSum;
+    U32  realRepSum;
 } seqStore_t;
 
 seqStore_t ZSTD_copySeqStore(const ZSTD_CCtx* ctx);

lib/zstd_opt.h

@@ -46,11 +46,18 @@ FORCE_INLINE U32 ZSTD_GETPRICE(seqStore_t* seqStorePtr, U32 litLength, const BYT
     if (matchLength >= MaxML) matchLength = MaxML;
     price += ZSTD_highbit(seqStorePtr->matchLengthSum) - ZSTD_highbit(seqStorePtr->matchLengthFreq[matchLength]);
 
+#if 0
     if (!litLength) 
         return price + 1 + ((seqStorePtr->litSum>>4) / seqStorePtr->litLengthSum) + (matchLength==0);
 
     return price + ZSTD_getLiteralPrice(seqStorePtr, litLength, literals) + ((seqStorePtr->litSum>>4) / seqStorePtr->litLengthSum) + (matchLength==0);
+#else
+    if (!litLength) 
+        return price + 1;
+
+    return price + ZSTD_getLiteralPrice(seqStorePtr, litLength, literals);
 //    return price + ZSTD_getLiteralPrice(seqStorePtr, litLength, literals);
+#endif
 }
 
 

lib/zstd_opt_internal.h

@@ -75,9 +75,7 @@ typedef struct {
 MEM_STATIC void ZSTD_rescaleFreqs(seqStore_t* ssPtr)
 {
     unsigned u;
-    
- //   printf("matchLengthSum=%d litLengthSum=%d litSum=%d offCodeSum=%d\n", ssPtr->matchLengthSum, ssPtr->litLengthSum, ssPtr->litSum, ssPtr->offCodeSum);
-    
+
     if (ssPtr->litLengthSum == 0) {
         ssPtr->matchLengthSum = (1<<MLbits);
         ssPtr->litLengthSum = (1<<LLbits);