pass entropy tables to optimal parser

for proper estimation of symbol's weights
when using dictionary compression.

Note : using only huffman costs is not good enough,
presumably because sequence symbol costs are incorrect.

lib/compress/huf_compress.c

@@ -216,6 +216,13 @@ size_t HUF_readCTable (HUF_CElt* CTable, U32* maxSymbolValuePtr, const void* src
     return readSize;
 }
 
+U32 HUF_getNbBits(const void* symbolTable, U32 symbolValue)
+{
+    const HUF_CElt* table = (const HUF_CElt*)symbolTable;
+    assert(symbolValue <= HUF_SYMBOLVALUE_MAX);
+    return table[symbolValue].nbBits;
+}
+
 
 typedef struct nodeElt_s {
     U32 count;

lib/compress/zstd_compress.c

@@ -1997,6 +1997,7 @@ static size_t ZSTD_compressBlock_internal(ZSTD_CCtx* zc,
         return 0;   /* don't even attempt compression below a certain srcSize */
     }
     ZSTD_resetSeqStore(&(zc->seqStore));
+    ms->opt.symbolCosts = &zc->blockState.prevCBlock->entropy;   /* required for optimal parser to read stats from dictionary */
 
     /* limited update after a very long match */
     {   const BYTE* const base = ms->window.base;

lib/compress/zstd_compress_internal.h

@@ -76,6 +76,8 @@ typedef struct {
     U32 rep[ZSTD_REP_NUM];
 } ZSTD_optimal_t;
 
+typedef enum { zop_none=0, zop_predef, zop_static } ZSTD_OptPrice_e;
+
 typedef struct {
     /* All tables are allocated inside cctx->workspace by ZSTD_resetCCtx_internal() */
     U32* litFreq;               /* table of literals statistics, of size 256 */
@@ -95,7 +97,8 @@ typedef struct {
     U32  log2matchLengthSum;     /* pow2 to compare log2(mlfreq) to */
     U32  log2offCodeSum;         /* pow2 to compare log2(offreq) to */
     /* end : updated by ZSTD_setLog2Prices */
-    U32  predefPrices;           /* prices follow a pre-defined cost structure, statistics are irrelevant */
+    ZSTD_OptPrice_e priceType;   /* prices follow a pre-defined cost structure, statistics are irrelevant */
+    const ZSTD_entropyCTables_t* symbolCosts;  /* pre-calculated symbol costs, from dictionary */
 } optState_t;
 
 typedef struct {

lib/compress/zstd_opt.c

@@ -32,11 +32,15 @@ static void ZSTD_setLog2Prices(optState_t* optPtr)
 static void ZSTD_rescaleFreqs(optState_t* const optPtr,
                               const BYTE* const src, size_t const srcSize)
 {
-    optPtr->predefPrices = 0;
+    optPtr->priceType = zop_none;
 
     if (optPtr->litLengthSum == 0) {  /* first init */
         unsigned u;
-        if (srcSize <= 1024) optPtr->predefPrices = 1;
+        if (srcSize <= 1024) optPtr->priceType = zop_predef;
+        assert(optPtr->symbolCosts != NULL);
+        if (0 && optPtr->symbolCosts->hufCTable_repeatMode == HUF_repeat_valid) { /* huffman table presumed generated by dictionary */
+            optPtr->priceType = zop_static;
+        }
 
         assert(optPtr->litFreq!=NULL);
         for (u=0; u<=MaxLit; u++)
@@ -94,7 +98,15 @@ static void ZSTD_rescaleFreqs(optState_t* const optPtr,
 static U32 ZSTD_rawLiteralsCost(const BYTE* const literals, U32 const litLength,
                                 const optState_t* const optPtr)
 {
-    if (optPtr->predefPrices) return (litLength*6);  /* 6 bit per literal - no statistic used */
+    if (optPtr->priceType == zop_static) {
+        U32 u, cost;
+        assert(optPtr->symbolCosts != NULL);
+        assert(optPtr->symbolCosts->hufCTable_repeatMode == HUF_repeat_valid);
+        for (u=0, cost=0; u < litLength; u++)
+            cost += HUF_getNbBits(optPtr->symbolCosts->hufCTable, literals[u]);
+        return cost;
+    }
+    if (optPtr->priceType == zop_predef) return (litLength*6);  /* 6 bit per literal - no statistic used */
     if (litLength == 0) return 0;
 
     /* literals */
@@ -110,7 +122,7 @@ static U32 ZSTD_rawLiteralsCost(const BYTE* const literals, U32 const litLength,
  * cost of literalLength symbol */
 static U32 ZSTD_litLengthPrice(U32 const litLength, const optState_t* const optPtr)
 {
-    if (optPtr->predefPrices) return ZSTD_highbit32((U32)litLength+1);
+    if (optPtr->priceType == zop_predef) return ZSTD_highbit32((U32)litLength+1);
 
     /* literal Length */
     {   U32 const llCode = ZSTD_LLcode(litLength);
@@ -135,7 +147,7 @@ static U32 ZSTD_fullLiteralsCost(const BYTE* const literals, U32 const litLength
  * to provide a cost which is directly comparable to a match ending at same position */
 static int ZSTD_litLengthContribution(U32 const litLength, const optState_t* const optPtr)
 {
-    if (optPtr->predefPrices) return ZSTD_highbit32(litLength+1);
+    if (optPtr->priceType == zop_predef) return ZSTD_highbit32(litLength+1);
 
     /* literal Length */
     {   U32 const llCode = ZSTD_LLcode(litLength);
@@ -176,7 +188,7 @@ ZSTD_getMatchPrice(U32 const offset, U32 const matchLength,
     U32 const mlBase = matchLength - MINMATCH;
     assert(matchLength >= MINMATCH);
 
-    if (optPtr->predefPrices)  /* fixed scheme, do not use statistics */
+    if (optPtr->priceType == zop_predef)  /* fixed scheme, do not use statistics */
         return ZSTD_highbit32(mlBase+1) + 16 + offCode;
 
     price = offCode + optPtr->log2offCodeSum - ZSTD_highbit32(optPtr->offCodeFreq[offCode]+1);