ZSTD_updatePrice

lib/zstd_compress.c

@@ -88,6 +88,14 @@ typedef struct {
     BYTE* matchLength;
     BYTE* dumpsStart;
     BYTE* dumps;
+    BYTE* matchLengthFreq;
+    BYTE* litLengthFreq;
+    BYTE* litFreq;
+    BYTE* offCodeFreq;
+    U32 matchLengthSum;
+    U32 litLengthSum;
+    U32 litSum;
+    U32 offCodeSum;
 } seqStore_t;
 
 static void ZSTD_resetSeqStore(seqStore_t* ssPtr)
@@ -97,6 +105,16 @@ static void ZSTD_resetSeqStore(seqStore_t* ssPtr)
     ssPtr->litLength = ssPtr->litLengthStart;
     ssPtr->matchLength = ssPtr->matchLengthStart;
     ssPtr->dumps = ssPtr->dumpsStart;
+
+    ssPtr->matchLengthSum = (1<<MLbits);
+    ssPtr->litLengthSum = (1<<LLbits);
+    ssPtr->litSum = (1<<Litbits);
+    ssPtr->offCodeSum = (1<<Offbits);
+
+    memset(ssPtr->litFreq, 1, (1<<Litbits));
+    memset(ssPtr->litLengthFreq, 1, (1<<LLbits));
+    memset(ssPtr->matchLengthFreq, 1, (1<<MLbits));
+    memset(ssPtr->offCodeFreq, 1, (1<<Offbits));
 }
 
 
@@ -184,7 +202,7 @@ static size_t ZSTD_resetCCtx_advanced (ZSTD_CCtx* zc,
     /* reserve table memory */
     const U32 contentLog = (params.strategy == ZSTD_fast) ? 1 : params.contentLog;
     const size_t tableSpace = ((1 << contentLog) + (1 << params.hashLog)) * sizeof(U32);
-    const size_t neededSpace = tableSpace + (256*sizeof(U32)) + (3*blockSize);
+    const size_t neededSpace = tableSpace + (256*sizeof(U32)) + (3*blockSize) + (1<<MLbits) + (1<<LLbits) + (1<<Offbits) + (1<<Litbits);
     if (zc->workSpaceSize < neededSpace) {
         free(zc->workSpace);
         zc->workSpace = malloc(neededSpace);
@@ -213,6 +231,12 @@ static size_t ZSTD_resetCCtx_advanced (ZSTD_CCtx* zc,
     zc->seqStore.litLengthStart =  zc->seqStore.litStart + blockSize;
     zc->seqStore.matchLengthStart = zc->seqStore.litLengthStart + (blockSize>>2);
     zc->seqStore.dumpsStart = zc->seqStore.matchLengthStart + (blockSize>>2);
+    zc->seqStore.litFreq = zc->seqStore.dumpsStart + (blockSize>>2);
+    zc->seqStore.litLengthFreq = zc->seqStore.litFreq + (1<<Litbits);
+    zc->seqStore.matchLengthFreq = zc->seqStore.litLengthFreq + (1<<LLbits);
+    zc->seqStore.offCodeFreq = zc->seqStore.matchLengthFreq + (1<<MLbits);
+ //   zc->seqStore.XXX = zc->seqStore.offCodeFreq + (1<<Offbits);
+
     zc->hbSize = 0;
     zc->stage = 0;
     zc->loadedDictEnd = 0;

lib/zstd_internal.h

@@ -83,6 +83,7 @@ static const size_t ZSTD_frameHeaderSize_min = 5;
 #define MINMATCH 4
 #define REPCODE_STARTVALUE 1
 
+#define Litbits  8
 #define MLbits   7
 #define LLbits   6
 #define Offbits  5

lib/zstd_opt.c

@@ -64,7 +64,7 @@ FORCE_INLINE U32 LZ5HC_get_price(U32 litlen, U32 offset, U32 mlen)
     return lit_cost + match_cost;
 }
 
-MEM_STATIC size_t ZSTD_getPrice(seqStore_t* seqStorePtr, size_t litLength, const BYTE* literals, size_t offsetCode, size_t matchCode)
+MEM_STATIC size_t ZSTD_updatePrice(seqStore_t* seqStorePtr, size_t litLength, const BYTE* literals, size_t offsetCode, size_t matchCode)
 {
 #if 0
     static const BYTE* g_start = NULL;
@@ -76,13 +76,14 @@ MEM_STATIC size_t ZSTD_getPrice(seqStore_t* seqStorePtr, size_t litLength, const
     size_t price = 0;
 
     /* literals */
-    seqStorePtr->lit += litLength;
-
+  //  seqStorePtr->lit += litLength;
+    printf("litSum=%d litLengthSum=%d matchLengthSum=%d offCodeSum=%d\n", seqStorePtr->litSum, seqStorePtr->litLengthSum, seqStorePtr->matchLengthSum, seqStorePtr->offCodeSum);
+    
     /* literal Length */
     if (litLength >= MaxLL) {
         *(seqStorePtr->litLength++) = MaxLL;
         if (litLength<255 + MaxLL) price += 8; else price += 32;
-    }   
+    }
     else *(seqStorePtr->litLength++) = (BYTE)litLength;
 
     /* match offset */
@@ -969,6 +970,7 @@ _storeSequence: // cur, last_pos, best_mlen, best_off have to be set
             }
 #endif
 
+            ZSTD_updatePrice(seqStorePtr, litLength, anchor, offset, mlen-MINMATCH);
             ZSTD_storeSeq(seqStorePtr, litLength, anchor, offset, mlen-MINMATCH);
             anchor = ip = ip + mlen;
         }